From e53b737604348d34cc4e5b6b7cf4c9533e33b262 Mon Sep 17 00:00:00 2001 From: actions Date: Mon, 5 Apr 2021 19:54:08 +0000 Subject: [PATCH] Bump CRD chart version of cis-benchmark (#1077) --- ...rancher-cis-benchmark-crd-1.0.400-rc00.tgz | Bin 0 -> 1453 bytes .../rancher-logging-3.9.002-rc03.tgz | Bin 0 -> 9721 bytes .../rancher-logging-crd-3.9.002-rc03.tgz | Bin 0 -> 26812 bytes .../rancher-monitoring-9.4.204-rc09.tgz | Bin 0 -> 243415 bytes .../rancher-monitoring-crd-9.4.204-rc09.tgz | Bin 0 -> 114200 bytes .../rancher-windows-exporter-0.1.000-rc03.tgz | Bin 0 -> 4543 bytes .../1.0.400-rc00/Chart.yaml | 10 + .../1.0.400-rc00/README.md | 2 + .../1.0.400-rc00/templates/clusterscan.yaml | 149 + .../templates/clusterscanbenchmark.yaml | 55 + .../templates/clusterscanprofile.yaml | 37 + .../templates/clusterscanreport.yaml | 40 + .../3.9.002-rc03/Chart.yaml | 10 + .../3.9.002-rc03/README.md | 2 + .../logging.banzaicloud.io_clusterflows.yaml | 765 +++ ...logging.banzaicloud.io_clusteroutputs.yaml | 4563 +++++++++++++ .../logging.banzaicloud.io_flows.yaml | 761 +++ .../logging.banzaicloud.io_loggings.yaml | 3536 ++++++++++ .../logging.banzaicloud.io_outputs.yaml | 4557 +++++++++++++ .../rancher-logging/3.9.002-rc03/.helmignore | 22 + .../rancher-logging/3.9.002-rc03/Chart.yaml | 20 + .../rancher-logging/3.9.002-rc03/README.md | 130 + .../3.9.002-rc03/app-readme.md | 22 + .../3.9.002-rc03/templates/NOTES.txt | 0 .../3.9.002-rc03/templates/_helpers.tpl | 66 + .../3.9.002-rc03/templates/clusterrole.yaml | 167 + .../templates/clusterrolebinding.yaml | 18 + .../3.9.002-rc03/templates/crds.yaml | 6 + .../3.9.002-rc03/templates/deployment.yaml | 62 + .../templates/loggings/aks/logging.yaml | 58 + .../templates/loggings/eks/logging.yaml | 59 + .../templates/loggings/gke/logging.yaml | 58 + .../loggings/k3s/logging-k3s-openrc.yaml | 68 + .../loggings/k3s/logging-k3s-systemd.yaml | 68 + .../templates/loggings/rke/configmap.yaml | 29 + .../templates/loggings/rke/daemonset.yaml | 124 + .../templates/loggings/rke2/configmap.yaml | 21 + .../templates/loggings/rke2/daemonset.yaml | 104 + .../rke2/logging-rke2-containers.yaml | 73 + .../templates/loggings/root/logging.yaml | 74 + .../3.9.002-rc03/templates/psp.yaml | 33 + .../3.9.002-rc03/templates/service.yaml | 20 + .../templates/serviceMonitor.yaml | 30 + .../templates/serviceaccount.yaml | 10 + .../3.9.002-rc03/templates/userroles.yaml | 35 + .../templates/validate-install-crd.yaml | 18 + .../templates/validate-install.yaml | 5 + .../rancher-logging/3.9.002-rc03/values.yaml | 156 + .../9.4.204-rc09/Chart.yaml | 10 + .../9.4.204-rc09/README.md | 2 + .../crd-manifest/crd-alertmanager.yaml | 4500 ++++++++++++ .../crd-manifest/crd-podmonitor.yaml | 260 + .../crd-manifest/crd-prometheus.yaml | 6002 +++++++++++++++++ .../crd-manifest/crd-prometheusrules.yaml | 91 + .../crd-manifest/crd-servicemonitor.yaml | 459 ++ .../crd-manifest/crd-thanosrulers.yaml | 4725 +++++++++++++ .../9.4.204-rc09/templates/_helpers.tpl | 29 + .../9.4.204-rc09/templates/jobs.yaml | 96 + .../9.4.204-rc09/templates/manifest.yaml | 14 + .../9.4.204-rc09/templates/rbac.yaml | 72 + .../9.4.204-rc09/values.yaml | 11 + .../9.4.204-rc09/.helmignore | 26 + .../9.4.204-rc09/CHANGELOG.md | 47 + .../9.4.204-rc09/CONTRIBUTING.md | 12 + .../9.4.204-rc09/Chart.yaml | 46 + .../rancher-monitoring/9.4.204-rc09/README.md | 346 + .../9.4.204-rc09/app-README.md | 15 + .../9.4.204-rc09/charts/grafana/.helmignore | 23 + .../9.4.204-rc09/charts/grafana/Chart.yaml | 17 + .../9.4.204-rc09/charts/grafana/README.md | 424 ++ .../grafana/dashboards/custom-dashboard.json | 1 + .../charts/grafana/templates/NOTES.txt | 54 + .../charts/grafana/templates/_helpers.tpl | 82 + .../charts/grafana/templates/_pod.tpl | 448 ++ .../charts/grafana/templates/clusterrole.yaml | 25 + .../grafana/templates/clusterrolebinding.yaml | 20 + .../configmap-dashboard-provider.yaml | 25 + .../charts/grafana/templates/configmap.yaml | 69 + .../templates/dashboards-json-configmap.yaml | 35 + .../charts/grafana/templates/deployment.yaml | 47 + .../grafana/templates/headless-service.yaml | 18 + .../charts/grafana/templates/ingress.yaml | 55 + .../grafana/templates/nginx-config.yaml | 75 + .../templates/poddisruptionbudget.yaml | 22 + .../grafana/templates/podsecuritypolicy.yaml | 48 + .../charts/grafana/templates/pvc.yaml | 28 + .../charts/grafana/templates/role.yaml | 32 + .../charts/grafana/templates/rolebinding.yaml | 21 + .../charts/grafana/templates/secret-env.yaml | 14 + .../charts/grafana/templates/secret.yaml | 22 + .../charts/grafana/templates/service.yaml | 50 + .../grafana/templates/serviceaccount.yaml | 13 + .../grafana/templates/servicemonitor.yaml | 36 + .../charts/grafana/templates/statefulset.yaml | 47 + .../templates/tests/test-configmap.yaml | 17 + .../tests/test-podsecuritypolicy.yaml | 29 + .../grafana/templates/tests/test-role.yaml | 14 + .../templates/tests/test-rolebinding.yaml | 17 + .../templates/tests/test-serviceaccount.yaml | 9 + .../charts/grafana/templates/tests/test.yaml | 48 + .../9.4.204-rc09/charts/grafana/values.yaml | 562 ++ .../9.4.204-rc09/charts/k3sServer/.helmignore | 23 + .../9.4.204-rc09/charts/k3sServer/Chart.yaml | 13 + .../9.4.204-rc09/charts/k3sServer/README.md | 54 + .../charts/k3sServer/templates/_helpers.tpl | 87 + .../templates/pushprox-clients-rbac.yaml | 74 + .../k3sServer/templates/pushprox-clients.yaml | 135 + .../templates/pushprox-proxy-rbac.yaml | 63 + .../k3sServer/templates/pushprox-proxy.yaml | 52 + .../templates/pushprox-servicemonitor.yaml | 39 + .../9.4.204-rc09/charts/k3sServer/values.yaml | 86 + .../charts/kube-state-metrics/.helmignore | 21 + .../charts/kube-state-metrics/Chart.yaml | 20 + .../charts/kube-state-metrics/README.md | 80 + .../kube-state-metrics/templates/NOTES.txt | 10 + .../kube-state-metrics/templates/_helpers.tpl | 47 + .../templates/clusterrole.yaml | 180 + .../templates/clusterrolebinding.yaml | 19 + .../templates/deployment.yaml | 192 + .../kube-state-metrics/templates/pdb.yaml | 17 + .../templates/podsecuritypolicy.yaml | 42 + .../templates/psp-clusterrole.yaml | 22 + .../templates/psp-clusterrolebinding.yaml | 19 + .../kube-state-metrics/templates/service.yaml | 36 + .../templates/serviceaccount.yaml | 18 + .../templates/servicemonitor.yaml | 25 + .../templates/stsdiscovery-role.yaml | 29 + .../templates/stsdiscovery-rolebinding.yaml | 20 + .../charts/kube-state-metrics/values.yaml | 156 + .../kubeAdmControllerManager/.helmignore | 23 + .../kubeAdmControllerManager/Chart.yaml | 13 + .../charts/kubeAdmControllerManager/README.md | 54 + .../templates/_helpers.tpl | 87 + .../templates/pushprox-clients-rbac.yaml | 74 + .../templates/pushprox-clients.yaml | 135 + .../templates/pushprox-proxy-rbac.yaml | 63 + .../templates/pushprox-proxy.yaml | 52 + .../templates/pushprox-servicemonitor.yaml | 39 + .../kubeAdmControllerManager/values.yaml | 86 + .../charts/kubeAdmEtcd/.helmignore | 23 + .../charts/kubeAdmEtcd/Chart.yaml | 13 + .../9.4.204-rc09/charts/kubeAdmEtcd/README.md | 54 + .../charts/kubeAdmEtcd/templates/_helpers.tpl | 87 + .../templates/pushprox-clients-rbac.yaml | 74 + .../templates/pushprox-clients.yaml | 135 + .../templates/pushprox-proxy-rbac.yaml | 63 + .../kubeAdmEtcd/templates/pushprox-proxy.yaml | 52 + .../templates/pushprox-servicemonitor.yaml | 39 + .../charts/kubeAdmEtcd/values.yaml | 86 + .../charts/kubeAdmProxy/.helmignore | 23 + .../charts/kubeAdmProxy/Chart.yaml | 13 + .../charts/kubeAdmProxy/README.md | 54 + .../kubeAdmProxy/templates/_helpers.tpl | 87 + .../templates/pushprox-clients-rbac.yaml | 74 + .../templates/pushprox-clients.yaml | 135 + .../templates/pushprox-proxy-rbac.yaml | 63 + .../templates/pushprox-proxy.yaml | 52 + .../templates/pushprox-servicemonitor.yaml | 39 + .../charts/kubeAdmProxy/values.yaml | 86 + .../charts/kubeAdmScheduler/.helmignore | 23 + .../charts/kubeAdmScheduler/Chart.yaml | 13 + .../charts/kubeAdmScheduler/README.md | 54 + .../kubeAdmScheduler/templates/_helpers.tpl | 87 + .../templates/pushprox-clients-rbac.yaml | 74 + .../templates/pushprox-clients.yaml | 135 + .../templates/pushprox-proxy-rbac.yaml | 63 + .../templates/pushprox-proxy.yaml | 52 + .../templates/pushprox-servicemonitor.yaml | 39 + .../charts/kubeAdmScheduler/values.yaml | 86 + .../charts/prometheus-adapter/.helmignore | 21 + .../charts/prometheus-adapter/Chart.yaml | 20 + .../charts/prometheus-adapter/README.md | 147 + .../prometheus-adapter/templates/NOTES.txt | 9 + .../prometheus-adapter/templates/_helpers.tpl | 43 + ...r-auth-delegator-cluster-role-binding.yaml | 19 + ...cs-apiserver-auth-reader-role-binding.yaml | 20 + .../custom-metrics-apiserver-deployment.yaml | 125 + .../custom-metrics-apiserver-pdb.yaml | 22 + .../custom-metrics-apiserver-psp.yaml | 68 + ...-resource-reader-cluster-role-binding.yaml | 19 + ...tom-metrics-apiserver-service-account.yaml | 12 + .../custom-metrics-apiserver-service.yaml | 22 + .../templates/custom-metrics-apiservice.yaml | 23 + .../custom-metrics-cluster-role.yaml | 16 + .../templates/custom-metrics-configmap.yaml | 96 + ...-metrics-resource-reader-cluster-role.yaml | 23 + .../external-metrics-apiservice.yaml | 23 + .../external-metrics-cluster-role.yaml | 20 + ...a-custom-metrics-cluster-role-binding.yaml | 23 + ...external-metrics-cluster-role-binding.yaml | 19 + .../resource-metrics-apiservice.yaml | 23 + ...resource-metrics-cluster-role-binding.yaml | 19 + .../resource-metrics-cluster-role.yaml | 22 + .../prometheus-adapter/templates/secret.yaml | 15 + .../charts/prometheus-adapter/values.yaml | 152 + .../prometheus-node-exporter/.helmignore | 21 + .../prometheus-node-exporter/Chart.yaml | 16 + .../charts/prometheus-node-exporter/README.md | 63 + .../templates/NOTES.txt | 15 + .../templates/_helpers.tpl | 66 + .../templates/daemonset.yaml | 151 + .../templates/endpoints.yaml | 18 + .../templates/monitor.yaml | 25 + .../templates/psp-clusterrole.yaml | 15 + .../templates/psp-clusterrolebinding.yaml | 17 + .../templates/psp.yaml | 52 + .../templates/service.yaml | 23 + .../templates/serviceaccount.yaml | 16 + .../prometheus-node-exporter/values.yaml | 143 + .../charts/rke2ControllerManager/.helmignore | 23 + .../charts/rke2ControllerManager/Chart.yaml | 13 + .../charts/rke2ControllerManager/README.md | 54 + .../templates/_helpers.tpl | 87 + .../templates/pushprox-clients-rbac.yaml | 74 + .../templates/pushprox-clients.yaml | 135 + .../templates/pushprox-proxy-rbac.yaml | 63 + .../templates/pushprox-proxy.yaml | 52 + .../templates/pushprox-servicemonitor.yaml | 39 + .../charts/rke2ControllerManager/values.yaml | 86 + .../9.4.204-rc09/charts/rke2Etcd/.helmignore | 23 + .../9.4.204-rc09/charts/rke2Etcd/Chart.yaml | 13 + .../9.4.204-rc09/charts/rke2Etcd/README.md | 54 + .../charts/rke2Etcd/templates/_helpers.tpl | 87 + .../templates/pushprox-clients-rbac.yaml | 74 + .../rke2Etcd/templates/pushprox-clients.yaml | 135 + .../templates/pushprox-proxy-rbac.yaml | 63 + .../rke2Etcd/templates/pushprox-proxy.yaml | 52 + .../templates/pushprox-servicemonitor.yaml | 39 + .../9.4.204-rc09/charts/rke2Etcd/values.yaml | 86 + .../9.4.204-rc09/charts/rke2Proxy/.helmignore | 23 + .../9.4.204-rc09/charts/rke2Proxy/Chart.yaml | 13 + .../9.4.204-rc09/charts/rke2Proxy/README.md | 54 + .../charts/rke2Proxy/templates/_helpers.tpl | 87 + .../templates/pushprox-clients-rbac.yaml | 74 + .../rke2Proxy/templates/pushprox-clients.yaml | 135 + .../templates/pushprox-proxy-rbac.yaml | 63 + .../rke2Proxy/templates/pushprox-proxy.yaml | 52 + .../templates/pushprox-servicemonitor.yaml | 39 + .../9.4.204-rc09/charts/rke2Proxy/values.yaml | 86 + .../charts/rke2Scheduler/.helmignore | 23 + .../charts/rke2Scheduler/Chart.yaml | 13 + .../charts/rke2Scheduler/README.md | 54 + .../rke2Scheduler/templates/_helpers.tpl | 87 + .../templates/pushprox-clients-rbac.yaml | 74 + .../templates/pushprox-clients.yaml | 135 + .../templates/pushprox-proxy-rbac.yaml | 63 + .../templates/pushprox-proxy.yaml | 52 + .../templates/pushprox-servicemonitor.yaml | 39 + .../charts/rke2Scheduler/values.yaml | 86 + .../charts/rkeControllerManager/.helmignore | 23 + .../charts/rkeControllerManager/Chart.yaml | 13 + .../charts/rkeControllerManager/README.md | 54 + .../templates/_helpers.tpl | 87 + .../templates/pushprox-clients-rbac.yaml | 74 + .../templates/pushprox-clients.yaml | 135 + .../templates/pushprox-proxy-rbac.yaml | 63 + .../templates/pushprox-proxy.yaml | 52 + .../templates/pushprox-servicemonitor.yaml | 39 + .../charts/rkeControllerManager/values.yaml | 86 + .../9.4.204-rc09/charts/rkeEtcd/.helmignore | 23 + .../9.4.204-rc09/charts/rkeEtcd/Chart.yaml | 13 + .../9.4.204-rc09/charts/rkeEtcd/README.md | 54 + .../charts/rkeEtcd/templates/_helpers.tpl | 87 + .../templates/pushprox-clients-rbac.yaml | 74 + .../rkeEtcd/templates/pushprox-clients.yaml | 135 + .../templates/pushprox-proxy-rbac.yaml | 63 + .../rkeEtcd/templates/pushprox-proxy.yaml | 52 + .../templates/pushprox-servicemonitor.yaml | 39 + .../9.4.204-rc09/charts/rkeEtcd/values.yaml | 86 + .../9.4.204-rc09/charts/rkeProxy/.helmignore | 23 + .../9.4.204-rc09/charts/rkeProxy/Chart.yaml | 13 + .../9.4.204-rc09/charts/rkeProxy/README.md | 54 + .../charts/rkeProxy/templates/_helpers.tpl | 87 + .../templates/pushprox-clients-rbac.yaml | 74 + .../rkeProxy/templates/pushprox-clients.yaml | 135 + .../templates/pushprox-proxy-rbac.yaml | 63 + .../rkeProxy/templates/pushprox-proxy.yaml | 52 + .../templates/pushprox-servicemonitor.yaml | 39 + .../9.4.204-rc09/charts/rkeProxy/values.yaml | 86 + .../charts/rkeScheduler/.helmignore | 23 + .../charts/rkeScheduler/Chart.yaml | 13 + .../charts/rkeScheduler/README.md | 54 + .../rkeScheduler/templates/_helpers.tpl | 87 + .../templates/pushprox-clients-rbac.yaml | 74 + .../templates/pushprox-clients.yaml | 135 + .../templates/pushprox-proxy-rbac.yaml | 63 + .../templates/pushprox-proxy.yaml | 52 + .../templates/pushprox-servicemonitor.yaml | 39 + .../charts/rkeScheduler/values.yaml | 86 + .../charts/windowsExporter/.helmignore | 23 + .../charts/windowsExporter/Chart.yaml | 15 + .../charts/windowsExporter/README.md | 17 + .../windowsExporter/scripts/copy-binary.ps1 | 40 + .../windowsExporter/scripts/proxy-entry.ps1 | 11 + .../windowsExporter/templates/_helpers.tpl | 64 + .../windowsExporter/templates/configmap.yaml | 11 + .../windowsExporter/templates/daemonset.yaml | 69 + .../windowsExporter/templates/rbac.yaml | 78 + .../windowsExporter/templates/service.yaml | 15 + .../templates/servicemonitor.yaml | 38 + .../charts/windowsExporter/values.yaml | 44 + .../files/ingress-nginx/nginx.json | 1463 ++++ .../request-handling-performance.json | 981 +++ .../files/rancher/rancher-default-home.json | 1275 ++++ .../9.4.204-rc09/requirements.lock | 54 + .../9.4.204-rc09/requirements.yaml | 145 + .../9.4.204-rc09/templates/NOTES.txt | 4 + .../9.4.204-rc09/templates/_helpers.tpl | 200 + .../templates/alertmanager/alertmanager.yaml | 118 + .../templates/alertmanager/cleanupSecret.yaml | 88 + .../templates/alertmanager/ingress.yaml | 53 + .../alertmanager/ingressperreplica.yaml | 53 + .../alertmanager/podDisruptionBudget.yaml | 21 + .../templates/alertmanager/psp-role.yaml | 21 + .../alertmanager/psp-rolebinding.yaml | 18 + .../templates/alertmanager/psp.yaml | 52 + .../templates/alertmanager/secret.yaml | 166 + .../templates/alertmanager/service.yaml | 47 + .../alertmanager/serviceaccount.yaml | 16 + .../alertmanager/servicemonitor.yaml | 33 + .../alertmanager/serviceperreplica.yaml | 46 + .../templates/exporters/core-dns/service.yaml | 24 + .../exporters/core-dns/servicemonitor.yaml | 33 + .../kube-api-server/servicemonitor.yaml | 36 + .../kube-controller-manager/endpoints.yaml | 20 + .../kube-controller-manager/service.yaml | 27 + .../servicemonitor.yaml | 44 + .../templates/exporters/kube-dns/service.yaml | 28 + .../exporters/kube-dns/servicemonitor.yaml | 46 + .../exporters/kube-etcd/endpoints.yaml | 20 + .../exporters/kube-etcd/service.yaml | 27 + .../exporters/kube-etcd/servicemonitor.yaml | 50 + .../exporters/kube-proxy/endpoints.yaml | 20 + .../exporters/kube-proxy/service.yaml | 27 + .../exporters/kube-proxy/servicemonitor.yaml | 38 + .../exporters/kube-scheduler/endpoints.yaml | 20 + .../exporters/kube-scheduler/service.yaml | 27 + .../kube-scheduler/servicemonitor.yaml | 44 + .../kube-state-metrics/serviceMonitor.yaml | 30 + .../exporters/kubelet/servicemonitor.yaml | 151 + .../node-exporter/servicemonitor.yaml | 32 + .../grafana/configmap-dashboards.yaml | 24 + .../grafana/configmaps-datasources.yaml | 38 + .../grafana/dashboards-1.14/apiserver.yaml | 1734 +++++ .../dashboards-1.14/cluster-total.yaml | 1841 +++++ .../dashboards-1.14/controller-manager.yaml | 1144 ++++ .../grafana/dashboards-1.14/etcd.yaml | 1118 +++ .../grafana/dashboards-1.14/k8s-coredns.yaml | 1340 ++++ .../k8s-resources-cluster.yaml | 2582 +++++++ .../k8s-resources-namespace.yaml | 2286 +++++++ .../dashboards-1.14/k8s-resources-node.yaml | 978 +++ .../dashboards-1.14/k8s-resources-pod.yaml | 1772 +++++ .../k8s-resources-workload.yaml | 2034 ++++++ .../k8s-resources-workloads-namespace.yaml | 2195 ++++++ .../grafana/dashboards-1.14/kubelet.yaml | 2515 +++++++ .../dashboards-1.14/namespace-by-pod.yaml | 1429 ++++ .../namespace-by-workload.yaml | 1697 +++++ .../node-cluster-rsrc-use.yaml | 964 +++ .../dashboards-1.14/node-rsrc-use.yaml | 991 +++ .../grafana/dashboards-1.14/nodes.yaml | 994 +++ .../persistentvolumesusage.yaml | 575 ++ .../grafana/dashboards-1.14/pod-total.yaml | 1196 ++++ .../prometheus-remote-write.yaml | 1655 +++++ .../grafana/dashboards-1.14/prometheus.yaml | 1227 ++++ .../grafana/dashboards-1.14/proxy.yaml | 1222 ++++ .../grafana/dashboards-1.14/scheduler.yaml | 1068 +++ .../grafana/dashboards-1.14/statefulset.yaml | 927 +++ .../dashboards-1.14/workload-total.yaml | 1402 ++++ .../templates/grafana/dashboards/etcd.yaml | 1118 +++ .../dashboards/k8s-cluster-rsrc-use.yaml | 959 +++ .../grafana/dashboards/k8s-node-rsrc-use.yaml | 986 +++ .../dashboards/k8s-resources-cluster.yaml | 1479 ++++ .../dashboards/k8s-resources-namespace.yaml | 963 +++ .../grafana/dashboards/k8s-resources-pod.yaml | 1006 +++ .../dashboards/k8s-resources-workload.yaml | 936 +++ .../k8s-resources-workloads-namespace.yaml | 972 +++ .../templates/grafana/dashboards/nodes.yaml | 1383 ++++ .../dashboards/persistentvolumesusage.yaml | 573 ++ .../templates/grafana/dashboards/pods.yaml | 680 ++ .../grafana/dashboards/statefulset.yaml | 926 +++ .../templates/grafana/namespaces.yaml | 13 + .../templates/grafana/servicemonitor.yaml | 32 + .../job-patch/clusterrole.yaml | 33 + .../job-patch/clusterrolebinding.yaml | 20 + .../job-patch/job-createSecret.yaml | 65 + .../job-patch/job-patchWebhook.yaml | 66 + .../admission-webhooks/job-patch/psp.yaml | 54 + .../admission-webhooks/job-patch/role.yaml | 21 + .../job-patch/rolebinding.yaml | 21 + .../job-patch/serviceaccount.yaml | 15 + .../mutatingWebhookConfiguration.yaml | 31 + .../validatingWebhookConfiguration.yaml | 31 + .../prometheus-operator/clusterrole.yaml | 79 + .../clusterrolebinding.yaml | 17 + .../prometheus-operator/deployment.yaml | 145 + .../prometheus-operator/psp-clusterrole.yaml | 20 + .../psp-clusterrolebinding.yaml | 17 + .../templates/prometheus-operator/psp.yaml | 51 + .../prometheus-operator/service.yaml | 53 + .../prometheus-operator/serviceaccount.yaml | 12 + .../prometheus-operator/servicemonitor.yaml | 32 + .../additionalAlertRelabelConfigs.yaml | 16 + .../additionalAlertmanagerConfigs.yaml | 16 + .../prometheus/additionalPrometheusRules.yaml | 40 + .../prometheus/additionalScrapeConfigs.yaml | 16 + .../templates/prometheus/clusterrole.yaml | 36 + .../prometheus/clusterrolebinding.yaml | 18 + .../templates/prometheus/ingress.yaml | 53 + .../prometheus/ingressThanosSidecar.yaml | 48 + .../prometheus/ingressperreplica.yaml | 53 + .../templates/prometheus/nginx-config.yaml | 66 + .../prometheus/podDisruptionBudget.yaml | 21 + .../templates/prometheus/podmonitors.yaml | 37 + .../templates/prometheus/prometheus.yaml | 262 + .../templates/prometheus/psp-clusterrole.yaml | 20 + .../prometheus/psp-clusterrolebinding.yaml | 18 + .../templates/prometheus/psp.yaml | 55 + .../rules-1.14/alertmanager.rules.yaml | 61 + .../templates/prometheus/rules-1.14/etcd.yaml | 157 + .../prometheus/rules-1.14/general.rules.yaml | 50 + .../prometheus/rules-1.14/k8s.rules.yaml | 121 + .../kube-apiserver-availability.rules.yaml | 158 + .../rules-1.14/kube-apiserver-slos.yaml | 79 + .../rules-1.14/kube-apiserver.rules.yaml | 363 + .../kube-prometheus-general.rules.yaml | 31 + .../kube-prometheus-node-recording.rules.yaml | 39 + .../rules-1.14/kube-scheduler.rules.yaml | 65 + .../rules-1.14/kube-state-metrics.yaml | 51 + .../prometheus/rules-1.14/kubelet.rules.yaml | 39 + .../rules-1.14/kubernetes-apps.yaml | 210 + .../rules-1.14/kubernetes-resources.yaml | 103 + .../rules-1.14/kubernetes-storage.yaml | 63 + .../kubernetes-system-apiserver.yaml | 66 + .../kubernetes-system-controller-manager.yaml | 39 + .../rules-1.14/kubernetes-system-kubelet.yaml | 91 + .../kubernetes-system-scheduler.yaml | 39 + .../rules-1.14/kubernetes-system.yaml | 47 + .../rules-1.14/node-exporter.rules.yaml | 79 + .../prometheus/rules-1.14/node-exporter.yaml | 210 + .../prometheus/rules-1.14/node-network.yaml | 34 + .../prometheus/rules-1.14/node.rules.yaml | 53 + .../rules-1.14/prometheus-operator.yaml | 57 + .../prometheus/rules-1.14/prometheus.yaml | 202 + .../prometheus/rules/alertmanager.rules.yaml | 54 + .../templates/prometheus/rules/etcd.yaml | 157 + .../prometheus/rules/general.rules.yaml | 50 + .../templates/prometheus/rules/k8s.rules.yaml | 83 + .../rules/kube-apiserver.rules.yaml | 39 + .../kube-prometheus-node-alerting.rules.yaml | 41 + .../kube-prometheus-node-recording.rules.yaml | 41 + .../rules/kube-scheduler.rules.yaml | 65 + .../prometheus/rules/kubernetes-absent.yaml | 129 + .../prometheus/rules/kubernetes-apps.yaml | 161 + .../rules/kubernetes-resources.yaml | 103 + .../prometheus/rules/kubernetes-storage.yaml | 63 + .../prometheus/rules/kubernetes-system.yaml | 145 + .../prometheus/rules/node-network.yaml | 48 + .../templates/prometheus/rules/node-time.yaml | 34 + .../prometheus/rules/node.rules.yaml | 202 + .../prometheus/rules/prometheus-operator.yaml | 43 + .../prometheus/rules/prometheus.rules.yaml | 109 + .../templates/prometheus/service.yaml | 52 + .../templates/prometheus/serviceaccount.yaml | 16 + .../templates/prometheus/servicemonitor.yaml | 42 + .../templates/prometheus/servicemonitors.yaml | 34 + .../prometheus/serviceperreplica.yaml | 46 + .../rancher-monitoring/clusterrole.yaml | 123 + .../rancher-monitoring/config-role.yaml | 48 + .../rancher-monitoring/dashboard-role.yaml | 47 + .../rancher-monitoring/default-dashboard.yaml | 17 + .../exporters/ingress-nginx/service.yaml | 24 + .../ingress-nginx/servicemonitor.yaml | 33 + .../rancher-monitoring/hardened.yaml | 124 + .../ingress-nginx-dashboard.yaml | 18 + .../templates/validate-install-crd.yaml | 19 + .../9.4.204-rc09/values.yaml | 2635 ++++++++ .../0.1.000-rc03/.helmignore | 23 + .../0.1.000-rc03/Chart.yaml | 15 + .../0.1.000-rc03/README.md | 17 + .../0.1.000-rc03/scripts/copy-binary.ps1 | 40 + .../0.1.000-rc03/scripts/proxy-entry.ps1 | 11 + .../0.1.000-rc03/templates/_helpers.tpl | 64 + .../0.1.000-rc03/templates/configmap.yaml | 11 + .../0.1.000-rc03/templates/daemonset.yaml | 69 + .../0.1.000-rc03/templates/rbac.yaml | 78 + .../0.1.000-rc03/templates/service.yaml | 15 + .../templates/servicemonitor.yaml | 38 + .../0.1.000-rc03/values.yaml | 44 + index.yaml | 190 + 489 files changed, 110528 insertions(+) create mode 100755 assets/rancher-cis-benchmark/rancher-cis-benchmark-crd-1.0.400-rc00.tgz create mode 100755 assets/rancher-logging/rancher-logging-3.9.002-rc03.tgz create mode 100755 assets/rancher-logging/rancher-logging-crd-3.9.002-rc03.tgz create mode 100755 assets/rancher-monitoring/rancher-monitoring-9.4.204-rc09.tgz create mode 100755 assets/rancher-monitoring/rancher-monitoring-crd-9.4.204-rc09.tgz create mode 100755 assets/rancher-windows-exporter/rancher-windows-exporter-0.1.000-rc03.tgz create mode 100755 charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/Chart.yaml create mode 100755 charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/README.md create mode 100755 charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/templates/clusterscan.yaml create mode 100755 charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/templates/clusterscanbenchmark.yaml create mode 100755 charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/templates/clusterscanprofile.yaml create mode 100755 charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/templates/clusterscanreport.yaml create mode 100755 charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/Chart.yaml create mode 100755 charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/README.md create mode 100755 charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_clusterflows.yaml create mode 100755 charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_clusteroutputs.yaml create mode 100755 charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_flows.yaml create mode 100755 charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_loggings.yaml create mode 100755 charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_outputs.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/.helmignore create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/Chart.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/README.md create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/app-readme.md create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/NOTES.txt create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/_helpers.tpl create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/clusterrole.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/clusterrolebinding.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/crds.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/deployment.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/aks/logging.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/eks/logging.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/gke/logging.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/k3s/logging-k3s-openrc.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/k3s/logging-k3s-systemd.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke/configmap.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke/daemonset.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke2/configmap.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke2/daemonset.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke2/logging-rke2-containers.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/root/logging.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/psp.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/service.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/serviceMonitor.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/serviceaccount.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/userroles.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/validate-install-crd.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/validate-install.yaml create mode 100755 charts/rancher-logging/rancher-logging/3.9.002-rc03/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-alertmanager.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-podmonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-prometheus.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-prometheusrules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-thanosrulers.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/templates/jobs.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/templates/manifest.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/templates/rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/CHANGELOG.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/CONTRIBUTING.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/app-README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/dashboards/custom-dashboard.json create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/NOTES.txt create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/_pod.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/clusterrole.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/clusterrolebinding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/configmap-dashboard-provider.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/configmap.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/dashboards-json-configmap.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/deployment.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/headless-service.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/ingress.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/nginx-config.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/poddisruptionbudget.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/podsecuritypolicy.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/pvc.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/role.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/rolebinding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/secret-env.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/secret.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/service.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/serviceaccount.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/statefulset.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-configmap.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-podsecuritypolicy.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-role.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-rolebinding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-serviceaccount.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-clients-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-clients.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-proxy-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-proxy.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/NOTES.txt create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/clusterrole.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/clusterrolebinding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/deployment.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/pdb.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/podsecuritypolicy.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/psp-clusterrole.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/service.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/serviceaccount.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/stsdiscovery-role.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-clients.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-clients.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-proxy.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-clients.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/NOTES.txt create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-auth-reader-role-binding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-deployment.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-pdb.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-psp.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-service-account.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-service.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-configmap.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-resource-reader-cluster-role.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/hpa-custom-metrics-cluster-role-binding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/hpa-external-metrics-cluster-role-binding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/secret.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/NOTES.txt create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/daemonset.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/endpoints.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/monitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/psp.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/service.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/serviceaccount.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-clients.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-proxy.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-clients.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-proxy.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-clients.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-proxy.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-clients.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-proxy.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-clients.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-proxy.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-clients.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-proxy.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-clients-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-clients.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-proxy.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-clients.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-proxy.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/.helmignore create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/Chart.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/README.md create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/scripts/copy-binary.ps1 create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/scripts/proxy-entry.ps1 create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/configmap.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/daemonset.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/rbac.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/service.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/values.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/files/ingress-nginx/nginx.json create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/files/ingress-nginx/request-handling-performance.json create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/files/rancher/rancher-default-home.json create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/requirements.lock create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/requirements.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/NOTES.txt create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/_helpers.tpl create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/alertmanager.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/cleanupSecret.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/ingress.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/ingressperreplica.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/podDisruptionBudget.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/psp-role.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/psp-rolebinding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/psp.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/secret.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/service.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/serviceaccount.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/serviceperreplica.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/core-dns/service.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/core-dns/servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-api-server/servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-controller-manager/endpoints.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-controller-manager/service.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-controller-manager/servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-dns/service.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-dns/servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-etcd/endpoints.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-etcd/service.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-etcd/servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-proxy/endpoints.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-proxy/service.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-proxy/servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-scheduler/endpoints.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-scheduler/service.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-scheduler/servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-state-metrics/serviceMonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kubelet/servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/node-exporter/servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/configmap-dashboards.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/configmaps-datasources.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/apiserver.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/cluster-total.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/controller-manager.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/etcd.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-coredns.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-node.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/kubelet.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/namespace-by-pod.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/namespace-by-workload.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/node-rsrc-use.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/nodes.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/pod-total.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/prometheus.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/proxy.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/scheduler.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/statefulset.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/workload-total.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/etcd.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-cluster-rsrc-use.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-node-rsrc-use.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-cluster.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-namespace.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-pod.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-workload.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-workloads-namespace.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/nodes.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/persistentvolumesusage.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/pods.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/statefulset.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/namespaces.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/clusterrole.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/clusterrolebinding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/deployment.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/psp-clusterrole.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/psp-clusterrolebinding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/psp.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/service.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/serviceaccount.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/additionalAlertRelabelConfigs.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/additionalAlertmanagerConfigs.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/additionalPrometheusRules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/additionalScrapeConfigs.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/clusterrole.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/clusterrolebinding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/ingress.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/ingressThanosSidecar.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/ingressperreplica.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/nginx-config.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/podDisruptionBudget.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/podmonitors.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/prometheus.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/psp-clusterrole.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/psp-clusterrolebinding.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/psp.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/alertmanager.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/etcd.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/general.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/k8s.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-state-metrics.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubelet.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-apps.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-resources.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-storage.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/node-exporter.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/node-exporter.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/node-network.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/node.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/prometheus-operator.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/prometheus.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/alertmanager.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/etcd.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/general.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/k8s.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kube-apiserver.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kube-prometheus-node-alerting.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kube-prometheus-node-recording.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kube-scheduler.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-absent.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-apps.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-resources.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-storage.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-system.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/node-network.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/node-time.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/node.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/prometheus-operator.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/prometheus.rules.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/service.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/serviceaccount.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/servicemonitors.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/serviceperreplica.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/clusterrole.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/config-role.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/dashboard-role.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/default-dashboard.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/hardened.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/ingress-nginx-dashboard.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/validate-install-crd.yaml create mode 100755 charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/values.yaml create mode 100755 charts/rancher-windows-exporter/rancher-windows-exporter/0.1.000-rc03/.helmignore create mode 100755 charts/rancher-windows-exporter/rancher-windows-exporter/0.1.000-rc03/Chart.yaml create mode 100755 charts/rancher-windows-exporter/rancher-windows-exporter/0.1.000-rc03/README.md create mode 100755 charts/rancher-windows-exporter/rancher-windows-exporter/0.1.000-rc03/scripts/copy-binary.ps1 create mode 100755 charts/rancher-windows-exporter/rancher-windows-exporter/0.1.000-rc03/scripts/proxy-entry.ps1 create mode 100755 charts/rancher-windows-exporter/rancher-windows-exporter/0.1.000-rc03/templates/_helpers.tpl create mode 100755 charts/rancher-windows-exporter/rancher-windows-exporter/0.1.000-rc03/templates/configmap.yaml create mode 100755 charts/rancher-windows-exporter/rancher-windows-exporter/0.1.000-rc03/templates/daemonset.yaml create mode 100755 charts/rancher-windows-exporter/rancher-windows-exporter/0.1.000-rc03/templates/rbac.yaml create mode 100755 charts/rancher-windows-exporter/rancher-windows-exporter/0.1.000-rc03/templates/service.yaml create mode 100755 charts/rancher-windows-exporter/rancher-windows-exporter/0.1.000-rc03/templates/servicemonitor.yaml create mode 100755 charts/rancher-windows-exporter/rancher-windows-exporter/0.1.000-rc03/values.yaml diff --git a/assets/rancher-cis-benchmark/rancher-cis-benchmark-crd-1.0.400-rc00.tgz b/assets/rancher-cis-benchmark/rancher-cis-benchmark-crd-1.0.400-rc00.tgz new file mode 100755 index 0000000000000000000000000000000000000000..2c47d93b9ca7001e467109ca6cc55676410e248c GIT binary patch literal 1453 zcmV;e1ycGSiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PI>{liD^Azh^#$CVj>+e~#<$b|GmeZF9NduIck4?}2)@dH2aufE2kyiSxE&X;^TC73x1?nyrraME)7Xj$!F4x}M$pSPvD+v6n z6UB91cQhQvzpm>Rf8Fc;%~fyc4*H|f^`Jkva(g59=K6}cry4#)B153B+>>n;U+$F% zNT~=&C~3wW#yAl8YVL3#^3f6MEk{k_NuW1oDOXB|ErjQx9J4p1BYaajL_&Zj1dd~7 z*``9&KuUMb$`A&nQ#ASr2D^6Z+WszAI}nO5sEtr^%$B{5hlXnrQr!M14T0|)Mhj$< z>3hSTlrG0|I(3F<72=pd82W<8tJYa&c6*NN3|+UYx$Abm^|l@?)&5Tyggy|Ox7?2m zp*9@kQSqS0|8GY9;kN(xdxO!%|IZ?M&w)_jS3+r9#(DnPSQlLOW=PQMJPYZK*(5Tg zf+-pmX^!vlNk|d9L4X7v5ZEA+I8lx@#^J(j+`3~zCVn3Rv{da-xFJM=G_kdCZwoKBkgM^%061C@*T*)Li?=VXc z1|@XcPemKIA2mo5W7-&9&Y@by*SKV2>Ceyu!k4rXpfNCiF?fw&vN+y8`YvDFM32=k zg1skv96)l=8dh5xPngh$RcqM>wN|=14OGR&JzP31KiPeL0a#l5aY;z7d)>-CEI-?~ z6U>UI#&%HE%%EPP_bW=&xr0}6YFYYEPsgoM&!)HL2av6^dGqNu(H?Q?c@{exP(Lfa zmo=YeFECAKRZz5~yU?}Fm7vzG^fH@VAz}q%32$SDE28E!S z`cHp!eO;*k4ElqcOa12@a+dnfnl+~vg6y3=MZVW+DZJEux9UGzn%WXTAms1S47Ef; zlP*RM^HY_&9vHsv1+G%p0`qtm-es$rT$(>YSg3~84m-(ET*^Pt+GgzOHIuc4s^ZCp zZQo}Vr9j`7?rVEi|K%+tf9tejrp#$tLN0OSHjrQS!JuLO50j&Lv%i6A@_%o5Gb-f& z{>||6|Ib-KaPSVBtk+R9l)Mt~0wA=l|pYp4vI{r^T+`j+x!3n&RZPb1L*&Fr?-+x{Y`|c(F zpF@7y^ReSCnDc zVQyr3R8em|NM&qo0PKDFbK5x5=>GcgujnVw_a^pdQHPJr>ealejGgSJHiyg3%+^+F z%7)06gfU5Q08ox5&imV6;ovD!qI@N@E7a7)B7sJu(dY{ekdw%pA@2rkI;GLHcQPYf zx(gBpe|fOd@Av!1hll##e!pM+cQAZD{LA35e=s~gJ~|j4{-r-S93CG31^N$%LUT!k zB>XS^d-qix+@GY7C}NUG$|5mx06Zc|z^1N8qzurdtVa^bx-=4!1i=U@?9%nP|6@p` zH>+*yA(wPQ(a##zR`aQd1G4BwB*YQCF`#Q}sv06D9*)2~%n%U^Au+6N#KZ^!8YQ1< z-^QHHsgI&No%0b`aPF8ye-P>gEb(2+TrWt3L_P`FO>aIJqa=e`U>pM^0_{jD#HwwP z(5}bAm_-=Ltb2#V^cUoUvSPIIm2+)%PNaVwW1`@e~ zBp?wqn!=1uXTbtI7ES0h;ba^joSdHu7tYuXa`a)mfS>e$9e@)x=5!L^eTt?+?4&>mN8@d*%Fll&bunlORFyunJ&}{C|FYa8#B5M}woS{C|qF1E)A4 zNg%<@JWLo*muZ(}tA7VR%%}h=fWW_AynWN1Fdh;qQ41VY6;9C$2uGL`PL+%iAQ_Ai zVkH@Tppj&-UwC!V)F^2985RGDW3{IL1sU5wXYwLNBD8Gmd_@ zAfCHbWE}uWrm3I=h?5{VV*&LRBX~V|$K)ADfswR*oh3nVfgVRGM)2`hErQ<9k#p+f z2s#~Sk_5rh!%PvCo;pQs3ZK6?&W@r>9I%C!xR3}VZI{LC&W_`8Boa>&Az65if+bw( zxd{mbDngkD1}?x3{1YJ&2?>G)aP(LhV&p@_q$0bB(W8?E_;zly)>=hK1_6>ef>d)# z)SP-O5*}rV(4f5;qaJEjkTcZ{a@Q~b2?PTgDO3dt@Ma|P%)eX;;7J550*vOAvna$! z3b>(i1_4$2PQ%Vn*$0t$Gu8cD8qw=yj4r&MC>8;q!3~K@BPldXA~Ra4k(;m}U^hm+ z?kGL4su2ITMCkXS>L5Z7g&=&fugA|ZWP0=nUc!R~j5*P?nJCTqeNO7{T?;g%QkzFR z0N};R2nPLrXn%(oY9<{Fe|Ssn5Pwflv~A%;%*P8<8k}*}M3I6=s1``cdT}HR6uYFh zS{JTyks3m#ssjL!E`$+uUjCktprfDa;0&CpK*bR067(1^-~ zVPB>0%_!7g5Ktj8f|zj$Q4)@|J*R$+;D`PXeO>Va`CKuNj~WnVk>;b{6XXj6MVJ~6 zq<@VxHgXZ;2u_Tp_4-Ud^U_}m$q4nc9e9ziGQ}dbpaH6D(X@R-#u%(>7U&9g^}>f4 z3WN?H6R!gTrKHhRfVOze(ujrRNWyfb2^U^54lD{5*7}M?3^dY}*&KO57FuG8eYlxX zZ|3Z%O<;R6GVv~j1pOxG?(l5%u* z%o5#CtBjePnPC|yYFU)PKGU)xBn!>Wlp{3k?OBNhQ?Olm=91X4nMwIcf(+?4t-?DzZbf9=f&?iuqdc>R~bct@@5 zNp1s|*Qw7u(UU}6i(YDJV^<9&!?KqSM?b<(vZtheywsiMy&X`}^o4lGqI1TiC1?8d zXMtRcl)id25r5+>iAQ=6j;M!5l$bo@bWQ`D;!EL?fEf3|j!vmu;=hlAh3fE?;!*8W zWiz=09*N1A22@hagqzPskSyjisQ= zNE`#r^GIRqv36!PX?BNX3<+e@EoU9NX3Mmj*-M#-y6fjS9In|Kb%so4c0{m=%i%2 zoa#E@rxjpEgf$v`Wlz_@$okT%TbVDGJ;A3+-U*m+7M2aJg7Gq57=ueQWNLF|3~ntU z4ZqF;E7@PQcGt}emA}*2f6CHnFV-W4+_Ih8wII5QO@>8s%M>B+oga~YsJAkfZ>vfIe zkI-2PBC19m=;qb5x^T^oqw)m6{l!F|i{Uo7RgyCD;P^oQNyCd|GNGTK)6Icl-~mt{M1*qViJ- zM-v)C-#Eimz_al}6Xo>Xg{rHxfQZrCb)DA}u)9MBh&8@#?94o)5^iV^z!*)WP>1Fe z@)q0mG){w+Icex5BWpF1nbm2m(2Dkf$RT~F$1(b`{qz>_%t&IOxaWpmrg4UA>FLG@ z#+*iS0-gUB-TxMy3N+L2CJfi~w_(IHySJFF867gtS4}qCneD5LX%jb6mBtATB#p{g zY8d+tlUa1bpEFUoy*Q$M3#wY znc!m!WKBf_jl^-qF)-VxpFd}&ZIdu|70qWDgszyGJ>50UERl4(<$g6AN1o}sytE2@ zWT^Kd+taRxm2LVeP)kj2;rE2e63f&=2}wkz%70#zaZ01q%af54-MB@UR+3$E>loE8 z3L84LvG`hts)nHhLfq>g%z_;_*EI68Eu3=}pcA4bK9NL+9F*{; zEc!r06q1DT2%;ni9Dqz-rQEmDveCkaCjsh|5)xA#{kB_TI*^b@ZS=D&YAEziVRm~R zP>N%m*3p_p|LJ3Z_RlFw{ck{p`6ITuPF+Z1KW%WMAGSgTBR^&|k_MAZQ`(=*#|m0Y z2yD=oa^CcYot(2E3DFA(4NXrm^HW-AIY>N@B`H?+*%J|{3pmF}3d=?{B-5`Ok!edI z66@)&0QU)oED~skZqpI6-$D|JlOzyN!7moWF5oALky>tb2!f6GdDbpzJL|DQiU zF5CaZ!C)}h+W${c*4Y2|Gyi`wd{goQ?qKU@NmM-tWsTx?LyI4~^0bDSuoQiZUBybi#!b9E=29e`YJuU!A! zf1o_@b?bjPte*cl9vt+y>;EZAW&PLXAZIZpt`h;ii;0}=!*}C_L;<58!PU89?%x77 zhK}Nz-k#>0j)|_pcWDKbrg17*3iiX+l=w=edHv^}=6$8W+Wp_-rTE{$^KJa^NlH<< zlQ2IU&SAzXOEKTsE*kJNG+Yh$rIC;DCFRS$?>-Np*ErF zz8ft~P*!i7K&ISBX_9I-8HI#(`|F{wybGSjve5a}DU}OJ_yr1-&zzc~no9OB5(au6 zBRi2l6hWsdLMN@+$u(~ZDr_miIPbVF_xSVrh6UwRJILxeqP)YU6Gj^ix6o%HS+@YB zo3F;={$ybiEea#(l#Im2tu4g?1(N8-Zl0M6U%rf%UQ05CFJC&toO**{8CuM^tUyXhHj^?gy(m!ljM;tAv@ccJ)u&#J0bN9^z~yEquqS%%6fj=QT|;bwi5wQnkR$aGn$78@CzY)?PLiTprlkfQiOHSUmWk*e%7 zB4;D$^yY-Ch2A57Bpmhewg6?(&|DG?|e+SRE_TN*Kw)P)Bsr`pr`|q*sKWw)DLKH&5%inW--+&s! z)(qU5fm<_hYX)x3z<&=ju!a3My~anj{|=Y(zlOtY{>PJ)w)Wrj8lTYqn_lDA{(D^e zZ+eYQ_TP0f#%|2nr^Pnvw>9^+=HAxa+nRg-Cg$E!X<`3eA7nqe*9Sr=)rfl=n1XBU zzr*42Va5JC*yjH}QL$2lME-gk@cJN+0Y1au+cl_OG6)^RKb_sjHzThPwqe9aHY2YO z8pDWc_A~ntKS@r0(dUrg>Z8Ufm@cF@UFR+HTrwYpq>;U-G^>!mPp>Sk=%jB-FMdOo zsvX6n+N%%HM=Ps!^M1Cr>(+LCFx#~i`#pdK`#}A)uwx&%r>|knZp)_S4)(vjBf7l< za0mPU;9$75|DU3K1^Yj}V|?oX{F^%f#dLwK{lB&UxAydFTl;@&|9?BBh5gU3 zv6o$;c$W;2HTM7D`1q)r|FOOQ^NC8^{14uo|B>JPqnYF_iCbFdTxWUQ%kEA)$v?KE z_0tG$wfy*a@$&qa*C#K3%?!pjYo?@F-4Q9R3xE6p+h;l{0A=b*Ap6`qILzRA_VjCm8`<) zPnG_mr~2snWAkr*=v*HIL)RZ?ZQo*e`BO5vd_zMj??}n&C#}i^m9q+N+We>EUix_? zZZgw)dumE7iQoVHp+w_lDoz-`A>7XgqQB->EPL6zzcV4rt?L*$Re4Rmo-5{FJ}$;9 zAoDL3sdA2zFA7uzc%?rE^Ee8yrTowCNqq$Qe|&T_T$2CC+xX9ulsn3Q?bGJcK{>O9 zYt8c)y$(D}KLydr)~}I5OP8P5d>LxDs%$EE)M&Q6z`84HIt?nPOQutfQ>9iq<>1g# zi*)^VviQ#0J7QL9yF%A%uXWjI5CCllV(M1!jM)I|;>|0oZzajZypxN%``<>=7}8q0 zm}EJQRlK8QUK5K-@ekEg6U}PWIZj!In<3N4r@R^1=g*}@Q&4tAAbE`yXjwH~3qTWv zvZ(U93?R3ddMLeK#eix;Y(l4%S0Wi3h|Lh{y?y)Pv*oKUpyryAQm3}%`?zxNhAHMd z!u%R912*MIeB_t1f=%gW9m5GV3|yVf854$jr5@uA_{89rNw@d>B)x>U+;c4ptm=L> zXSMCNmT@qQ<>Hj`>?^4iM9VDUr9MCz%hl2>U6*y2oNOQgA2ZMj&??w^#~Wd4xUyBG z=&twal=`dQ@tSGZ$|b3ftYTZm1n+Pk?b3HBHo)C}Wy0LfU%0iqbme>=L(>CF-?wJm zN(8Ry()z;X-Rb4?cce!1wY=?!a<`jnOIOym>S&{jds{)Q63onDWsC8w8e@Jc<8s|6 zx@`^g|86xw`jG=7XJ;HI^iz`}$hXN-Naa?mwb#b2Q)jZFb^5(>^EwG|1m{WgLcEB~ z7aUf8g*^ZMd42QJ_f(hPCUG2Kh>;{gaXBwlEEomdT&bB)PgsRm_?V3TGu6+3s(WqQ z`TxUv1%TJZe+Pr5`@fF2_rE_)*}(rFK1Kj|_}CHPO2~CN9B9)~RtIH=+3O2a{qhwJ zq=~IwUi|Cg!^^j)mp{LI{ozMVC~L@WLz3I_jhD9a|2_fWHS+)HU^rNk{|84~`TrE< zA>{x4W3t2UZ!F;v3H+}ZovlUUo^jg7fx%-&Y@0^1Y3$bjCL_1scsLO@2rgHGux+ER zWgt9>w6A98ycS6PyPE9DN4@y?tA@GK$!;HFT)B7~qs9*t5*>Dy058ws%7Ap0EgnA1 zcTcJPjRgD}M)4S-zZDGiCkO&I5$X}cfel?gNMNu5_M?XgAIR2OA1vI4*|ueEY3u)` z@fgj(x#if(8(eq(>u4$dcX-g>-v9a}x1uVY8Npxc+lbVHGXL>M^WBHr5`$ z#=SdU%1}(|%IwYh^I7$Eo6l-7zRC)0=jfGBiqsrZSldrFACG&;WCPc=(QR#ZS)1L} z?U}UgnY8VhwC$O+?U}UgnY6sLvHuy9_qqUho&B%gRrY@l4-U5W|5KE&mdO9`PQ&)5 zhX?X8p4ckin&g>TxQs~6Qc96od$6E|d)J&dUh?dkt1UToTQ_6tW^CPzt(&oRGak&% zcuf1R+5T57d8f~PER}WnzemH$`QO2DzrVHrpQI=oxP1Lv>HE{AhZQ09AiJarX4Upy zGUS`B^sHX?nQIb7Ocn)|K+j`goPAy0B|$tR?sYOo9-&0NHExy18D|q3ph*jX?~U=5 zzTD?YBx#80@9*1#x92tr;Kis7_y_S+*6j4g=^nCQIcKtRbGdeZ_-!Jj-rRl1BGp^{ zCUjQIJR{K*3zlg}qY(@;`!Xb-MsR#|ba1qGTlpSvS8isPGJOhL(d8}c&!4Th{@N>g zv(ha8ZKl)%=zw+dzkf8S%Kzho?fpMbQdVVz)rVc01<@o40#z-O#dnwD44k^M9#eT`ccGcF4mTU?fme+^qbEjku_ith5VY&4z8>6ka@3q@@`dzKjKL-Tz^2{~0@7EKf@*?|UPr{5Cd^*UEol5~?3j{`U_D z)%)MJ^8cwyB^+G}{@m+sm{N%Tn-=X~e-VDg(!Bl?ft<4d+vRw#D(mdOgX2p6_u%<( zc(`5vPf<2JIb@ybrXyDAx+8u_qmENPw%QCnJqLqb$xPc3bRH)*KJ{ebg+Q6U zY!DdfHe%0q@+x3A9kcUvFF5bD)VLR|0y851fs3@uXOdk=dqO$-&^b^4IypZTKsZ83g;4Z@Gaa)AE3rV%5In{?ikRus&Z-G` zGr}c|al&|3RlSolD$Gk)v-H~G6@Gl}Wqjw-UjE=oIqA+vS3n8n;d#-zm$16{v=ZkPtDcm!}RK3fZb* z^7r{_FN6bj{xd=Y)%EYrFbL^1VjLf=k9F(+Xn0)l{|^t3w*LQ$s;Emlu+8bavo`g&~XJmy-+d zDHo1ArLw30Z_sz#@gKaW|DQgXO?&Ep=}$3_dO4sm@s!0op#chK&lNYZv*(VaP{?EbjHvjW!inEg*vVt^N4kyY6cASsZ7r#E6Ng0b#uQ#P~mW=gY zZ^1|DRm1MP@RmemiavN4NlpU#2m0yBFYs6xvj`)Zo=(tdsQHy>kS3%6i_*Z4>o_|* zAcOzo^O56RU0scdm^t5p(pvh201)4Y0#02a$v8krG0#(8y4utiB`3>cVF_3gD+@S% zl)4qm;Fm_OP}8WsjwEM(;whQuI3F?>1ICO=AvuW!5XeaV{_dpqF%H-wRP zdN5ZW_r(=VSl}Z!oUdP{{udCb>j{Owq#?SBeUv{NhN`xiZUK9H)n&T3rx-xBEzCzc z)fq9};r!QC z69I}tJxTpWbBwNVENxB0tLe<+E@dml_*D@zNv17OYQ#(~U1Z*b*@634x8K-R0i!qx z0&|pOLt3eZ_K-*`j9yRPF?q&OU?i8j(ikbh{(g>}Qy*KB=@!xkMu)HokT`DD@M?7z z8lyC+VFRoR7^&9n(Q1cLmMzg9r3#JAA@+yE=q5{=Qh-~{+b!g$=_ND!cral+HC*3k zcI)}+9WW}UytL*cbEH!5p{7^Lg3XlDf>|d{)HS@4JXta99HYWvt5$sUHW5uyDns>2Is96KW`P&?U?$j%4`{eFKRI`;HqX9UB3zyHNrcVa^- zR&1%KCZ`^9(}~cIJrb(MUwh8f(kC5zogcl10GYN`4z!!=Pz`VGiG>>LF@<-=S%`9m ziO5xg1!BRv1vR9*tG^y*Jkr7WV6hJb{`KPR8y)vHUb~3VqZ4ZTH{PV?u8NCqz5AH6 zIrWvbF;gHaZoJt@hgXaPej;HU;3iR;?j-eQ7bM&u;`V=GZ|?(*_x48cFP3Ob6M?0G z$HxNTU!N6z>{bI)uL@zwS9w@+PDv_JjcXrsoa@~IP6KE!efQ{3i*@rf6ru z*+&i2bh9}3nsCjQc6X=UHq_NF)FlJ4p^po<@Y0?G`S9jKhfI@*dS>%>jPeFCDn?3+ z4RvA^ed(-eQtfKPmWehq}v2P7=o7e#|5a1y%pmuAvuWx7&0GlJf z;Z!1!M6!@b>X{I}YCT>EDq@Zx&&E=%Cc+DT65)Xs+x;igzbjXh97f`qPEeg;*4g^OIsX; z5L&sR-qj%@-*YWjtYFkS9DHld2Ze{XNjUfxt_JS0J6ID1uA>{4x?5(FwGdP*@0=;d9f&5)ne3YpQ8?=R{I>A@lpCMIsf(A>B})HH(EQ1w{WF5 zof~5jt1wzMeXTL7PU0t!AwbqlT&545Px--NMyxGz_V~5G=~S8QwKFW>(i)9YzgP*A}CtsoQMU?7wGo zPic0S=vy@3DywIyVUAIZTu>n~^3d`Wn1&E)dugfR)yH!r{y&u&_dfEl8l7H+Q5Gz6 z(_p+SB;#a?SF14+^p8e8aSK<&!QbebU8x3)Z1=SrFsm^#4L#Mc2BS`=vr6{rLE1dXOaA$4dH_Nr?>ai9g z-TAomm>R~gZ>PW}J?i7zX#h`TzxA|45qc7RgS|e`t!R^x$vm?9^oFc$TQl4ck zR8UGg$Wg2B^88dMwSKgH!>fSZ+?BUI?u+dFX5nP}rCh`)db`kt7bV2h(jz*f<=Iu? z80^)q*{bpNMz4Sy^+ON6z;;=VX_*XmVHGH-?*~McyPW_q50<(w31qi0bxX@chN+rmYoT{3laG87-gL@Lt zfMn4eBdTp9HnGXrq-LyoOh~1Y)}%a<%>Jvf$H>okLe7}L!Z4M{V~Kvg8UYe-Rv5;Faq~#^rDy-a*2L0cs4jxdxj)BSH=>$`g^AJwctEyY=GUl(QSZf} zdaQQ)Rk<@}hV%JduRwCOsM?v(u~M}KO_IH^XMwh+_O@)xwrtD4Vfp_700960>(ML` H03HDVS&tBA literal 0 HcmV?d00001 diff --git a/assets/rancher-logging/rancher-logging-crd-3.9.002-rc03.tgz b/assets/rancher-logging/rancher-logging-crd-3.9.002-rc03.tgz new file mode 100755 index 0000000000000000000000000000000000000000..e1fa6522ef22fe0c91073d3d9d6ac12c5004c873 GIT binary patch literal 26812 zcmagF1CV4}&@MXdX;1gGZQHi(?rGb$ZQHhO+qP}nxP8u?|K1ljUc9#>c1BgMT@kCQ z5?^M13p?y5!jFHhFVY|6IwEp3dLoic5-zNay7Wpk`tpn>s`9K%(n?B9lFH_$I+pq_ z3bIz5A_nFbKh8hY;`W$osYBmdxIpxFzaFDbNt@O(SwCv~$~ZU2%s3o{n}*(WhPZwP zONgTQ`~drLwu^bzmbEQNRwD2NJi4@#P?N1y;L}PWvg9F@dDn?X z$@rnQ`Qx<|Pz`OS`SXjL^sG6)(OQjUv;D0H$;=`%x4rq+uPq2v6S>_apX5!kwRPj^ zVIu77XkcPu%uIgM-R>cWThB3V^$X@y$QJBz>J3rV<^c@3VBX?^&sb@6k&VWzvpZGt0m{-3*+MS_4QRB@?{u-z}0RCgMxaTI6ebXkEIJ|&FtRj6z&bMXL+qK3_x!N?Ky!)OL zqw#toEQwmvp}m{u>(we3^vrpqol1fHKv5QI?ZN+N5%|L>lgy)ENvWs5dM6_vD~(mX z`I(OX(tuaT*F#d%x`h@pb@^Up1GV!>KjnQN*8wpQ@yRPipwXxB*ECJ;74{a|^Prd+ zxVj6QNw}tH@df$@nbYPHwM7S@{INrg*L87O(~uV$+u32mNz2{faifcd^P;}@vKPXo zc3#Bt*O%+pBI8w3E6JcV7sX?g+}49vLT_sq_s4;ePLK;jEu(s-XYMB$`Fvr*diuM) zB0>ZJ^XeZU8`l=7~`hXgC5(q=(Al(!kfpRatKH zft&*82Nd@y==_tF*Xu{x(*cqx*L%&G3iAd))6ntij3vvvMc42mY7SEn_fyz!w_|7J zbgp9MFW*_r&-S84k2Ey(rA0nA8tG3JgkD~cM`7;KX|W2i%RnJ5A71;-!IX3aLf;*EYmD64y(5xJ5c^ZPA$fe7wh zmIc2*di_chRo;cWFBR;?c}g?s+7bNUqQSRlSduv>DU&G2`D&-F`!bmBWYPgN>1BOv zdpXB5vwkxkFVc=E|Jq>Z&0hpmgVe*c3%YY!ygd1Vyv`8^bzh?>b7Eep1W;D52T|^4 zx>+treqJLF5R>Z zzeml`;+bgDkHv{>(L(Pa*pMN^?X3!$9Dbv%%C>+oF2t3v5m=%9Lcp$6PWTy;nd4rD zQ0LadO8Xg3GD-Gl{5Ev?psV+)OK5NxtL5Z$A4n=KK$qexO0qjRt6Hj6{B7~|bJj=Qz$)rYkF$6q51!>?;MB3aPnTY z`Vkf8!s!S7uUP@d*E66Zk`ama-jW-8jCU3>6+4P4}z?)*q?{{Lr(A3wL zkx($R3<`z%piD*sq*doc&D22pZUnO~hs0-X2=-F3TmroSyf$=A4@rJil6LQ$^`H)W z9i!w;0CWx;W22?7rueV6+M38owa-P8*{b9YXs#28Ah1+FA4&0q{nj4^S&-uyTTc-T zffhFW35b5*&WaqgjE_w!L1aEAFhBsnD$USy<AJP zzq4yV_^uKY3c0$RY8|N)(KDip;rV#7t5(WN-KvjFt9w*NUjk@dDi&wXJS5&ZGCfi% z`MR55!hG`lhfKxIFbbzo?c1*i}pJHl& zS5x-rF`_(FgLM7?&55G5$WEVQBe9{LJ+vj{MR+(Nc{IST_=Bfl45@FWskKo2RE}rW zsx6ph%ZIX$pe0YdTr-ARgvV)4K^r?UG~SLvpZ}Bl&%lT~!{cF`XCCH`ZJ-Tns)=-b zoV*~u3i2AH7fslMM0C*7c|{r0Zf5`#<7OWa+h~tZAAqQ#%k^KOpXDAfn(hF1Td^4S z>LEr4_)+5U?i7a{j4&U!3QkRkF!|M%R69E=arEjdQaK|>Qtb|{%FJSxK&o&;=RgC> z@X031;`w9w+eJqEHV~Ic!Rf2`0}i___f{GOYv&XZtqziBhlAsTfy;}krxDk|eaTQQ za(pNCI$_PfxOyj$u?L-hD48BK%V-DMSA}U;x{&5u!$R^|HaI}`at?}82dx~NiZbBW z|8ay?BTQ7`FJYF2!~irL-kcc3*uUfR$#9h{ z?(LFhw-=b1c|lmW+^Z{ES%!pu_^t24dV-O#gVIHWHV;+h^WYx%9o#<6>12GAHVXEc z$MlZ&+qM~kylc)JZ3smYCTPq7MeR6?&^+Ca6zau|d2gH-mbexs=iyor z|3d9apQTvh$`V%FIz(nXZyMHe%Q|-Dg*U5)k3?H!%3*Rt6$d=(+-% z`?%>__)3%ivRT%;fr##TjjHN=8*AF?bXWblSxefI`xrVh&T{`4;)LU}0vXO(qOJwq z&rW>s;w}SZeLpM%V)ZQbY2c~=E_td}fUFfM1{t|J-y&#|ibrZ@RhHnteJ$0fDzcEI zCGA>oG#y&3h#$9rUaoj_;};U_11N6-tr>Q(T1T*fLL$q7!%nDls|qmp-mC;fBzM?= zHl&zHf+aThqUDyb9vIU&n;8DhED3!u)U4;Y)+)B;lA%~FQzK?L^8!uN@JLL;&u0%j zLf&W^;UEre)=yC+FfU*H%OX>I0xP8xbM&b8^4F~!x}XhTN=hE}S6cwm-~|z{g|1A_ z!AVW2PdBA2WY8~Ymb>ol&KU@WEZvqhX4ekG9E7eJQP^rTi|s}12$U@rAc={qqq$yu z1+5CZ%TUxxG7aG*E-qHcjK{u-KER=VZt@z~BRo{k-*iwjuIfpgO=6%GsqtVJCg~MCLcOz-DjkR$tHr zgPe$ph@fXjsd5(hgv*ccEArKfOtC!jWaiqJ5Kqu)p73$*@CUiU?i8voAPug?rPKm` z0)DJf)@EyH$usaMPeTg7tDqQi<#!VV{IWQpiEhN$1&)9@5kpKs(cma(C3AbEGeP5$ z4>8M=i+Nagu#)H4Wy~qvD{K0>{UVH$o{49+iKYE!gtO~~eP|1OOq-Kx#=ZBY)&b2T zaFEy-@U(Pu1pKs+poOq$t5fXOxDzQo+vV{L*%|QcbsPU%shQ-8k>ejvQZdYiYk?ml zKzrXep?MWCv)w@Po3^3e!)fKF<(q#iElu`qlMxMiR<8KQUEqatsRyZ`r4N>eXM9bS z%#v={&q)u3hiz2(bxoVtW8GE_8w~C1E?!%EqQ$tlv3uxS!r)$t zCCrO9ZO{Y;;6MUyQs5Aj5TC+k+Spak`vt#3o>G9`WXkdcv`^|z+J=5Ob1hwW7L$*C z!=DS6aY?}OYL1RR{qr+mo56!{k}+CP_=IPgBD}}=k6!_a$db0~g^1}&KHM!{N=47rTE%9%n8 z_Nz++&xnL#*#hA4BtGzjGP(ZsqVl=G=ldkGfP`W^k%xCu@zEyLPdA6%;#@6vuO|k9 zM^ATaKF@wZ6jq-_Fss*ywhOQMmQM1bB{ff6i>TXiqtg4cN0W3p<$I;TizppP8Osj4 zRD0pV3@~=fFe_PCzUF}WQ!$d-t#Hs8j}ENphZz_7Os$BI47t?1vVZqv;1XZg*FlLM5!cK-0SUm?9y)E=oRC%lWI+mGUVx zda14BXl&G-kgFF&44kGdR9wMXMrS4`AP}Yn&j8yu;3G#A<_$E9cz40*$YP9%{fnr*K&=}=Adh_?=@8&YCS+Hs$IGP1@BYqW0ts*rjb?Y9Ff$0 ziG16Z_?X#bfosN>p?>llXv1D^ z%La~(yT6h3dX)vn)}HE{%ECZ?>0H1pn$#7M?FK20pGLo-=8QHoZKAbr^#L3f{;7Ze z#2-Dw4ILEzSrSK-$Bp_iax5+MRg%>q^u?Qcj9?1h6rHJBQo5|9mSkX9M#dCxG3jMj zu9;y8SX|1%Z^pdhK4)*)nG+chX(W({OLrQ2;i7C}SfMX1Zd?S_F+=3hLy@c&!;8G3 zUB@;)HpEaJ(#he`huPzZg|UTBOMDX$D?tT zWyhm-e=Vkld`b>NCNhD>q7?^A@LrI^^sn9oOE6P!E@C)LCBEn%(|UlS2xW9%MUZq4 zH4cBCx5r0gyEwCT7b9PyCChrCFds9mm}6W6tm@9!I)VM}Pk6HyqmO6=K7o|GkYlp9 zgiKVf%J-=L12)tlGP!VNJ~*?YQCZKoJN+}?0g?T6{27sBO$JgIS-&e*L4&upc=$q5 zT0rs^yX@t}HH%>@peL@ne`-c4ShMk4rxVRl>k6C7lP;iQd&sKdS`M0iArqi;9L_7p0Opk@Q(_e3mogj8b0>R=<#a1$C1TC{s6-~jwB@oR7e`OYe{b3V-0Q2V`+VA){Xt9 z&iVB@O?6ZX3LE$&oH}Psvvv3nqnafyzkY*Qd0U5qn59MLyjjJ0$FXfe*{4U2VTadu zbw{rdd9%^(xmM`pXqZ5?lCgt&Itq~MMp64ZrWA9zNuQy@aZEw~<2A>qqpK-4S~;;O zhJydL`d{mCpmllfE~uRx>|3KRw{r`(uhZv%)Z8Cf-+lRb`>wD+d)UNN(RiTZ@y?uu zFiJXuITlJlo3vlq18e|cB?rNbEdt+HTIrO%G4vRGB9gXcX9gkX$K-{X>bTBFZ1!v~ zg)jZr=!}|cGdjTnNAC0$f5uzBRY&>><;oB*EwqgUmF1lxYx-F@yL=`D__^-R3BTGm#F*_+m_HlMK_E zL?Y~q@J9ZIbwB5gjDOFJ-NJ5ae`5}h-KN*$o|h}egmg+zohsn(Gu1Y;xO{FA7m%SR za~k)ysIUFiMRJ>AmVX$X%5Mr&!kT}WFPS=zA!jEtfy-!gYUEWfer1|KS*AEjm#3yk zo2QJoCkAwhuH)^%LP*`@V}`r+t8rc7<@Yimvn2?JJCPo(`xfbQ#2^nC$fZ=;hIO5R z(Z>1t4>YO)i@}D&orQNoi$>k6$3Ze{j~}wHJZuO8O1N7%gKfPc9k^d>9iRh|h^hCa zDkN(7>Pj+PNgYrH^{&MZW>dwVE@cuPI@-5`OmE0oOJ&_}$y^+-^A77}=OQ-u2& z)*ftAC&VyzA_s{`^H5THJ?pNyfSgV#Afv1U$e@lKh1tJFmytmLeuZXsPv&+zM;3S^ zf@ATNG=of{8yg3Ma9E;DWm9OECr#5juPYlHqeSK847>+My1}}_5^|#43$Y7@RvOOJ zTNn%E{ptShQQ%?S3L0Pl`NtcN-$<6HHuiMFm3htv3q`FaiRT3eeP=#ZkWLaqPBux> z_m^PhIW82mo74Z@_;5ak={IB-uZE%Saj?osc_*c7-8r%gdiZ2ES<%Knr_@w3M^Mu!-@Y{W4-#mlxWY_U)a$=Ujw+mj z*f5V60*GWmmc{&zPyb?qJ3Fe_Mk4eX^cxE4f5T=jhj{VkZ@6|H3Y!W0aSnJ}904Ew zgK$)Kukm4)KGsESbmrG14M`nIipxTdHISX@W^cV3qs?}Q)CAC0Kf{GY_TlP_!Oc5+ zNm^$FKjHs`Nn9Zc$2Zgfjnci2&U+cxy9N#86K2E8A!}zIdaB!#69Zc42BdsTD(VJL zp%Z~7lpp*X#dQmzWC~nR#Ci|(rwZbw;%85qEp|NP4S`NYpO4pj{HVn}t@ZnY5A$uY z;7qOCcO#`cxXFtdhM#^(knN&||H|*9O5hT**LWT=2y8kC=h$&c=?lLUN_o$8ShbfZ ztU`}{V5$yh%kqFfkyGX)B#wSu(*qkjSjieHXh}WQ-CeWHV6O`Rxv0+^zF$rZ8Q{?a zhEd1{Rk9Ve@s_sn5@zel14+T_x#0w77&yTsCqx8?xk7Pr^6&Teq(BSzQRP&?0iD5_ z(VoPw31-rb^$`LVlZZ9Gjw|XJ#@{&?awE079pZG7RP;Zo%({I2PjFkz+Am2vef%rr zx!>+CT0d(y{jmE%6&cxx*X~_~>8jBdJ59*Y>}20*)VPA4m7xI{#4YN3_La@7&A) zA?+sJQpri2vm5 zFc!aT8Z^uFVHk?7JeG~%dzWqx(lv-c+J2$!s$5R*d;VcRoGh%{6IM%f>vH9_N9Gtg zVz4Jqs7&6zAIq!9uJeUl>)HY!4tbdV?$7SzE8z4V0H@~yyuxY}VN(-(vBXwJc+dRh zBXZ(OOtte;%->(PlghAU12*QyHmN6zvMKgNjb?n6r&fDQZe{j5bwS3m1>P}M#1DH) zwdJXt0ET_W*y&f$1H-=`9Mp+&o+p`I13Sa%K5t*I<#)BOA&6w2304i+bNLs|B^@{m zS8g99LW<3>F9gp9tNm3^4OVb$1v(?npx%`J^;bcch4)&+8%X7;(W z6L!Ql!AEF?EsG}sB3z7D7I9! zwZ-o6#O_G4o&D;WQpWbKZLIe9Tg=t%O^0!Hx|l1>WnjxZ;sdgc%~Jk|{ZN6|EV8|<#|g_Fb7+as1s&UCfo zG6T!qCVbg3E_El&)m{scHYxWMvF1WEivOIfCR|jdb||O$ygXm^M$uFiZWAdfo=O(X z%9SzVg^{_?mqY_^p(@BY<~I=X_j}aABfF%3t!zDE5|+OU6;JpR4zW{|Fni^@lg)06 z`Vm8CGC0p}&k;$YahP1^_r!s6Khz1+gCi=3lc~D0dAIXi7++N|6`ldwwMBrC6{CG5 zDMfJEl51)!Q@gTj{(^>wHsqR3RqrAH+3~^;Oo6r;9yrM;p#Xn-H+QzkXi@YUF3QeX zE98tJ%_|`Fyoy6=@JbpEpAY<#V()%zOc5#Cm3TOxj|h>jBw7?Cik~Q9UNlZLK!QwP z4mWOeeyS7WD_&xLZFmLk<|pgAjPpEW~VX?=oIay$=Oz|HW}2|0)}A8G;Bm%C}NcIp&h)dr*XK;mXGHACknx zz)lqv&+&fQ!}Xn3*dd)Z*864mp9iSylq@!NKykjz`1`rSh#w4I->tC9aM*KS0X#TM z`fue>)P4%31)=1?&_LlyxS!IfU2sufL7C zybqTYyY}njL*P>#m*|D(?Lq*?}xUnb3zy8chwuJ85`C2d^wKDES$oj7a{<8iadGLK)XLr|F~Z{@C%`5` z(jsrILsCV5;J{_6){H%sU&NW9?!T)r*@G%kFFFV+Jd|O zMw!fxuReJA5aQhP=}OAz*0^W_6b^>R@BV2wVXo=$-6(|YeIirY{R#LdYO0cJ5}TEY zHS>e5`m2t+=5|Zn#v_zG$1pZtZ*LLVjm&q!+nRHzdzLY zS+BAcj^RArnH2B{BMuvKFmcA^JUir@WGr~GM3nx)Vk09>9^rUJo^nIrQ^PwbDz{d8 zfE$>Q&B@I%zYc?HD4iD9$&z21J7z0bwYEhQPEG^{FGLdZI_%un-pwz$C+X0o*r@hP zir?*qdYnDwPa{opYsdaLq~d4*c$of%|E&qMBDof|s*DBs;+363g3~d95DjdX&PK?uIGejKA)9dp_6)3?vZ!qr9?n0$^n!$82MGtyWE_##0;Gzn$3Q zirs=Or!?L@bwX%sGR(QQTdAT{U-&H3*Hre$_fQzBE*B8AlmS*UQQbXfx&-K~Efp~B z*3v}3%<&6c^dxvBUb4`f&LEC7O=86)j!2#1yc}dnszzBN>iSQy`RUtz4)Es_5 zdunhX)#-wQ?O_~H2sfMsoL8#lUzg+-1sHXi@aWGA?hF?G>*7=sc_hvTt#B@IVgD1q zav750ljq7E&bnT!$U91GTJ|*eOqRc-z0*6F_7n*NVQ-lDWbL4=_q{|ugN7_13CF}9 z=Q@&;p!3}3PB0}j7h;@w$}{@?{)U)=em{=D;tJ&x@Jo<1oTd5CmLjvJ z_z2y?pGwJNy^3Q^wb$Q9%t$4n?ts$YwVf@-pXlCQt7iI{e3>Rze?FTQnw9An7&mIm zWw3u>#kjdx+y-fuwx*sMj4IYvG|Z#N?31mK0(6)~>^b0U`o;Dfg@-X<*~9|yyIr0T!K^h>hRCls%l-26UoblpD0<`5$z*; zU-m`QoJ%OZCN}llf`YJoVk;Q=LkfZr+(b(r9=jR@!ph3y7HJwk0=OoqiNKy4i- z$V>Z$IXwnhnqhRJ8@cjhKO3(XK(eN2T0r&F1;TqRlmg-Vy{B(3C#nesW`L%ZQE0d6ns4`YJIWrd6($vD`NR;Ck4&}G zF$w<_OwkF<+cl1t6^A`3?{yCA4aEA}_>od-ot$s}pNN*>nj;}<0#ggR+iP!?2v3~t zDYaD3lVKr>&aKT$0T$WqsqUUxu=UL!t{{*=MP&M7P2tsBlcm=vxd4lRsIj>r7`h!veS9HLEyp0&d%;`qfGguice|j>Ox^YJc;NI+_(9J4P}cHTP#v03X0Cg4Wowbn z*U-??(z+3P@7xk=hqJyC(h5Xw>stLL{kU}b1!7(oI;l8=`MG?7sX#9L{#vksH>iwv zT<3HZ2q)blOg@7yMs9NiDN`DTkl5V3ue%V!QDuTMN`f{}Hq5LMrEI7L&0>Mei zA1e;gzsh2Opglg8Fsn8`WOBYQy=i&5tc|1$gz4;cp*)*%Z$g-x^he`0K93+-LV9H3 zUwvSI@b}4ddbyvGLS~-s`zvNXJ&3DSbtAZXY?laYbZ~yo$v3vrwpwT+XELJVJZkAR z+p=R&EWI^Vx^xY?Sw7{>JAHT+R9{{Naeq(@`dLnazE#oEDC96z>>EC>DLT>dOgVi0 zCBA@hC+SWUTr7?!wx$VA8KGJ0EQ}fDvXPnQpv!D^eVja5 zZ6>J|!?Ly)!A#-HakB?yraOHA_G6`vqH}pPG8cuodJw;{w!j`J)-e3c~<+AG@7vaXWZ~U_4baco31q+MKKbzL zQSwsGpt6%}Y+bS^^dISkrnK*y&fQ8H4}qpj#@LYps_yEQy>O~=Hn)uD`Tg+*iwL|6 zc>ce6EJS@;GsZyTtr!`;tG0^0TVJV?Iqh)e$a>s>@EQQj+JPecR%BZCQ9kzF{C%i^ zd5;$e)p$KpDWPiH=863L=;%R#HCl!MdKM`|+csf`(=?6~IRoWsLmgli>7xIUR%G8^ zrh|p*yt~xV1UVFLZP!Xl&43oq1_I*1rLvbPnS0fQ=@A+1xwCxt)+GqpV)Y!1?P+ET zvC%Lkq|s)MW#4Q~5bS6`=m%gTP$=PUrh{j=Oa$_TFUAUFq4F-o!GH5P0@JG=y}(kzIv3Jqz-`a|o>(YdirMCQ+<-fEgU9;5D;%uCm! z#cI%R2Z-g1b`}TzsZFAjBaLJWmGhgB0b-ZjK9?f=1J&atU|Bxg{g~%uP1r75fCtU8 zoH|SR|AU|s0epoosk3p;e&GgQ)eJ*qUN+!@9Rvm0ED1kep2Cp+n;~$ON<^VrYJMGn zTuqOM)ypNgXxipC!}`-osjnsyIRC>@*|7gNDTUvlCzj_e8;JN_bUzbsQ2GCtl!7Z$ zvQKL4KFE3A{hqiUim#*}BXgFF_jF+Gol~G8sMFFz%OnTYZygA=><{e`9;xx`Txh&U zD*yeQ3f{5D274HGy6Pi<%ktGp5&%XEc9I9CAb%9>f3s51L|_~vL+}s!*)NLmf(TX5 z2PAz3e1O8dE`jS=cnj5KC_(DSGycRcTk8knRP#=y+1Uhh-p2Im_clr>NWe$m3H*NF z910^i#@&-D6JRONjs1~Mu(qwtAgUvZLZzwH8--ZXoE=dCeuT1!l1?$WW0XbJS&({{ zEA9w~@7T!}NA{zrlwritu6;iZDx-WgpIR!_H7+u%U{~KUuKbX|D_TI281_pJnM0IJ z8i`HmLKLiPxwz(3nvPBmM2zqOa@~OF7vtt_5<#6d0Ihzse=zPcgV|B6_86)zTz@cO z-+(308wTW?8P58Lg0sEe$k~6il)^CHqU^_zdK97#2pCfL<;ll>^YF?Zi=2#@8hsEt z9`};XDiS)u=>-*>57hZ9MYYRr%BC-}>S5?$Ifx{^vKnd5dV8m%d|rZd(=ehLjcA>{ zc2qXHpIkJu4JY?LmMlqzlW@^9{5gvWDq-$bLAGO()qux0+P#`TK+!CsOUoXA%@S;1kZN1sEnOA7G;a)FS!G@Gyy7d zWQIWjotJ&a`_&!eQH9ivcc$^nJ_muzQBJ)DxnH^3L%^66Y~Bv3qE^Cc}!7g&;Ek zMf0oAX}EahhPTIFQBe+fvR>CMP@DW`uu8`98pn6zxMc|ksOt?DyfD7PDFIN+zn)JI z%B;FPU-Ccq%BsHj96WfEBZHoN^ z1*EsY!B;a+BL!7b1w$@3#iCgJ@$z={e7d9u68p)b4I}YN0QG$sPSSG*!&VTV8CA## z=qu>-k!ULU^a1Vb<@95&cYb%V;za?L+Uf)1OScC~PNJscl7sF0YM-^r3C z*F%*dl9K~&ffrKuF)o4$@bCn-11@-l+eBb5_|asBC2?hsJRS*q7$vw^C3V`2?_hQp zND@VH;yhqi_p@p7NH(&! zfTFM@PXrMJNj3Z3o+!5e+Uv-Ie3)R9UsJ59PuCrUq5>tATo#}*B(4+(6L4zM zHe)61B>1jt#ko)g9q6{2z`(K$!(F}^8E&^X(M7&%}fo#SOyoRR2fT{+S15JUqx zC6=0Eh1qKP()s*wR1I}9UNUjEg=<@0A>54kgkjR`I2w8Lni78psRWOs`1H{hAxTR1 z1YC$B1-w!3DH864I++r}V%yhY3jTx*MIqACaM*bDNV2A={j021-uG2EL)hn81y)RT z`}=2jMbzVCI>i$;+`cWK0-T~>9;1Z1xeESZ12MvQWRcnO)UrvS>W`Vog5}P1g7C)m zr&c!L=wiM&8O;~;@h$z|1F#APtT-ZFLdhMzhx56FwtkpcgT{J9X{B5Epe!QD!U#^@ z6>+*=;>;G(B+x&)W~-(9A!V$EYV)?5_qTeP5XpH^`^7$go~TSzd6baEGgUt@9E$mjpD zi#w=9(56@+z}Exk+NkvapFW7V7SWZ$-5apFi7O>63TNstwMAnqbhU-iRcBEd$7ffP zZ4nu6N2G2_+oOykaw56Jq!h-?ftoPi^6h&1X7HkwTBQ^`dMF6WT+bZ%a zKsKR3{s$alA>?iYw(pWInJluXDncb5qib|9B9vkg)%4p>#RAtEogi|hJ>g8b7Bo&D ze~3l!0PtSHU^h}asn`2(V?!>$+IEVhQ)9UQub zTHG_b=HlD~v2HBl|0|$mP$QeLWB>+YeAo}M^+r#sWd4Jt-g$DjKXl_{I(YP(%xfv6 za7}expDs7jCX&h4n#+vECT0@@#s^_sKI#Ow*n}^Rb-JQ8^eE@ynq)!~+V0mGkNZw- z;V5@^s{A0Y(Ny_XLjCY$IJV$|i_8sORnX~{x1egiX`ZwGfnyHlFvvmsqifTbc9+A% z--&dw7UV$b2nxU7`MI^~7i-IyE|mBQL?>)}R1Zp~4T8PJ_d<7%_o~i4%B)S%OdJpLsZ1BKB=!8CG5FgJNa%v`AOMOoK%&wD8?m zJm!r*y9Q)=PxEK2H1WWGA&1srAG{v>M{ux6{KQojP6DgjecPEz60ROE{9Ulwr=9f$ zE#cmT+e8pTI4rcD)NGX>qhdP~X{_g-ijEI}afS$|)p~aKo8I*>Xrie`_?r>G*cdr< z;!xI7uRo)ebIy%yDV&v8>QyhFgc-YRmmEXXxBy0>1(CULYah=ojj7)}?IhK!2Q zA+moe2xCNT+w2z80QzzHii+t%Q~&<#x2Hqoc`&{Q^U=Gr`Ur2UeH%X4nLihi=z4eU zM1e16uQzxZbG8@b9#a@K2lPbW;6*`}M*sn*|Hi{4cOx{fFuZxPX zcbr|3_xWBIH6qU7oE9;P>WQ8Fq&+$B$->6RAm&ENw3m$`$mA_weLx#`D}v^Rg&_S^ z6o9GeQL}p>bvhA-c4{52e4lp(H%1xHzUCV7!#F#T-Kw4%t)Gj;3eA6jtNTi|ac-x$ zW@_5#_Jh|MR(d=~i=Xg)EF$@My8Ic)Gg9ycH##@1J}MT*Lp`35;+o4gdyoH)N0*|K zx~4g!PQl8gfJSIh?TTrLV)hp@b}FvYI~5`F19fn{FL)72m8T+|E6p4KOXjXl`zp*` zu;;_Di#htw!`gl-3TpA{N9@+nF?kUaGTy{qF*!u1caQkQPyvTX=3Oteja*Y&V(?)0 z92p1^_+t=TQzC>8#L`eMe-W>c#5-uHcDxw7JpHwq@G;Q#jc@kMejJ5R$YXB&AEMPzE{jXr;wUSECP&$rn?_|3 z_CF>%!yv+}+z5Ejb|n#>SI#YN)St;1w&xJU?k4h$izlwbQ8!p(Kyjva)}*DlghN|v z|ICKYqxf9O;sXTVaD9fk(a)f+MfrL5KirI-ukc6eO<$K`uKS)tk_aSVhW!TvV8F0g| zwgh{Qy-ltEJK+0B&6d_zIWu|=6ZI$4zN`lDnT1vk6E;M)4!4=rL?8=K8{p%i+hgAK z(FnhvKSe!D^wx7=2ehGs!0s_-F$mGaloR(9P;rmVB1&|sa{2xNlFC8XL9lM%gU|8V z9Za5=_ewW~DLYPe_lGIqsFl#ZRuwc2J@YpEa`%UNw3n*(SAkky^Z20kY-h3d!l73> z%nnkx%8_TvwF}Hs&{p*mo&t z2KEL^VW8eis5$0+@J@7xSr%w7@q?lKRS-Bl;zoZZL-<$jq&@Jlmiw9*t8vX{Lez4f zwbDR9EmNK(m}i{)oY;$!xa{Gb9#BMMFGy^|kc%`^4QLWiaiK$(4ef}Mo|b7m$CkNT zLmIJHq{!Fy6aB&RSMm*+ThT?UE#}wy1cIHuq~J5W*~p&(v=gFlP0~vT{T4 zZM*&C)DPjdOc&L6d?j@3D4M&n@p#|BrIn~&6=pLo{AubG42+A_J)_dP<61YDME7@? zMbPs`?v9BIsFYicq7I-pE%!*8y5AXrrA+5WIqyNb>I5nHlh@F0$fZuckGWxLnYS=x zUN6%}HhJ;8NC+x#K;KLOjVx034bM(t7qfe^Dah;1D?C5Aaxsl1j`IWP7>m;Q^}J71Dc@4Pm;R=fQVBbxu5W##$&H(j`rP zV-pNQ8c081Wh*E8LnJk*D!A?&Q1HDuW0AbnEGm+#`G;-JOg>*C+5#v4e2bhA68AkkOU0*c&TtkKbq}UM2}0~i%~^HqXF9ugptuctpWgP58+pPV7qAal z&o7wKpxG-z2OG-cX2b3vdN1b9v@#siys{>}lt=AjKH&aGdi@2mF?}hYyQ7HSo~H=F zBk5~WYc3cOpF0E=X`j2DAQEe?hcqGz5#E0u0XzcwzDcb=r%9|meQ@Qs-+5_?JKp!< zX2*hrTwgk4G(A^EsIk#b4)T-Y{(b#bO^?&@-u%6gL;Ec@MX0s3hx3!3WZXg zl2`g!NXE}a^JwaR3=<>Z@#`4Me``?#Qe=}?;zU8qQD{rS)yQWF?g^<-bQgn;uw<`| zzDG$M4wZV4TlebaULO%kFqcLa?+p=Xs5L!M%^mEH&)+}@w0ETdf?|)h;`OuTXy)l9 zOzEm#$a*r_r&IXN2uE=8_ILGloy>&?4(Dd4j)D3~*dI=83?YmK&#gQC>SaYGcop;) zDRAxPF$5BE4Q8tdO|aTTBvLK0B=yIqaGP=e6^ho$X7n-`^T*dlN&L4TBvfuxbjqat zL0&osnBiFZ*ybm_pbT=xQ)@#-)0^OmC+uxFMw|1AU9>Vjr}33fF=H!u1~F4PNgZHY z*x}d-aJK>w=>Eyfpv`G;XA784U*CQHt*22v;#E}2qukX40zs;8RA<&-5nb)mwGulf zQUg(bEq@zW<&QwiCd3Y{FjE!_drYBie=qjM9|1Gam&yqajEwqpfSxzjnLg@{sC}Pw z(tm>;0Q{zniSxcE>0hymG z-{)C6DuOeZqy2SJ?@v||>{cFyP}T68oGCp7(+XFZXD~#r)|O?94;lAw+h8E|E2HH1 zlm_5EX$qz?HjMHm8PI%yeot^$d)&<^H*=lx!yv}cbx&;r_;3MuS_*-n2)YMp@TE=U zrEgF=IZQ18=H7usb~nlMe*@Acm8&sIg$e{ydV+E6ulFW5lC@;h=3-J)6y)M&Fvn7I z{YWjtTx^|IE5lqT!@@M()*wSe;S$YMRL|5DeDS9RiHAjrn#+ohIOsN7yPYvYx+8+@xe+%emf^tG7!;F{o{qXgQ|~LvcUcV;dp52c zrc{)F?n7GnO_YxuW%R?S1!0Isb`}NMhNy1;C}ACCF6gjfWBbUXlBq9_ zeo2xWuq4^p6M(+vmagvevviKQGdBY#RJ#-IT+~CLE1d;$U~`{Mz-nj6Z2E`v+(6@W zHDX4dG>RdaxRS7~y447bW<@99ueKq2?ZbTFxp)rlQ(9SxYrRs|l4ml1-@9=dHCFn{ z_YAe+ZKz3NQYC_INT2okmE>n#@`iy58dC#v6FTkg$&?H z=Ru*+#c=~|d>uG}1c~83PvwTF-R1v zQ3(!gBk8`J_^EKR(rPMZ1f#xs+k~0JK4k6L9v10j>D5s~eDk$)KBCB$C;xyGnWqRu z7y_$74F#)RX1__4?<_K@^pcXxLP?i%bgA@BE{nVLB@HFf&O zuIla|-PKi3?^S!P`+6WITy;8WF&cEzoVE~}b>eh#Z=AV_I5~v9t9jd!YA3GfarUs5 zU1MrR%FYGojvsHJ3hcFJSy7U!QEO5^kbO~^`NoP>$$Gjt|CViy7H-ovwAQH2kfqSx3iznLp$(HL3?GpqbK$yPyZeDearLKwfBk4fDx2STUS0&m}ZNfnZR@RvRT+ed1Z#-5P(@pEhLOK0V>W<$%c}8m32R* z*!~=!)+DV$J$}Dyjegik(KJ%rV&$dkBb6tcPW*kZ>tmnc@0_Uq%L~Sv$bN@?IHttt z{ut1kNa4$xW=7D9@DJzXw7%pdpEcuZKl;+N?8VO9 zg%c^naO=a5k!P6?T{fGTO zg&M%+7;!C&z@Ob$`C0%xmPw!XWQ#>U=bl*ayM4x%*B;#NkpB%65BgD8slX6(N!0!p z0N+lNGFoFmX{eu2T>y0BpSzr!8zWrh^9PG;VTYrLfA)X9o@D2WN@5YwDwG-ry#orW z`(HR8f&mEXLK6sF;$yLBm5gF7vx-IJ>tf&9{}w(cv_+ zd(+l2JJEROAj{E+sTMI3>3lNm)_iL|w~!MiwX#e^{6^Rg#JrDuO+>Pv{Q%2EGn95v zzB~^AQBNV?2xL6RRYtmQLc2LRqh-qROd0kPCjSslA-vpe!CgKW$`$h{ASjwSzhhkqJq zm9fN6;wjCwL*fxq-2x?K-VPb(z`+SsyH>#$>}Mw&+)gyI$;TcH9v$flK1vbOy$x$*aE@0tbsSUlV zg*TgFKm*LZh_*;2+g|m#Z*b$^Tw}V0zCeYDDp5a>P*6|ZVnj~G`0|CG{dIS(k#1xTUd~IWzdwLV_fMz)Gh?Il zr6zD2xsJgIudLob6#DjKPHF%b40MzZ9geG6)RqhH%(frakO;QM(1E_`<@WvcOZUFc zEXJO;C;RP`&r_TjJpaae%5CSRSEir;C-pvnpG?dcWl-5za3$ll$>lbdzAWH86C&4! zXxJMK%&hKA1dT)=Xugg-O$s9%#xdNMQ0Hne=Tx!%NY-HK{R8TDvgiFvuRn0W@uGQy zGokzk<}Y`ACfYSNaVYnV3bhZE(IqIRh}GJW1;EsTdI*)89xuCflI-4>em{Q5JOk4d zK+1rlpxr_PzIupGoXBBK2&rPB?)KebYUg~V!xYT*^`@3_D2(VlBff+_e>!#@b2Ymh z|8gA++m5`J;s3hRKPU9Cyr8?gnddo+v37d3hPcJ+5IVl|M%S`7G#{;VcU2KG_BroGHGx+9uVtEt3~h(^KoE>e6ay8Gk9 z;beEMlD&>YKfQj}h^mPK)S6N(`Mh;_yxaH8@5;%S-QtQi6K6k_QiiOfr&G32q6_x7 z)^+!ND8#q>1$v8k_SUfQSXCayMMR4D3UfTX8;2mXdNiM+4KZnn0}!L%0w9x(kF?M% zIRt5auqzDf=8#O&4fH&b{2J5YBdlpNo$Dlio}OST;WX=qWNI+hw*8x**f9t`T0#)V zDLhEre+(f}gw%%GFG8^bT)9riHS)nQ)4;4}9mX|rq-ooYMk~jSl zWVB3S7ZinVhass(jq8u*nw;0Wlq+x~n;}^}$Xv4P9r-s62cmsjNUe1SHIm*kp zb~cxaCorB~J=*?_0r;dD?Hf*WR^qm0EAA7~os!>^shhTcT+`tV(oe{^@h$8EXcnnh z?v{dUWp4IG`UQ<>0s5^BlHvv6Z_0_c zR5WXemkd%a?GnODQKsf@dfhl~d*yTk`c+P#9}U@+o8)7^y?Iah@|FF-+)~LgLx{slD`P-Kd#&q3Qv4=nA|Z&-|M#tl3xpuY5?P_{>P*u0}VNkVun0e^Ac!txur6F;7cCz1{ zmZh_LouY*5vYkm)v#oZ+TS$%nT0ec>pq=&-X+2dxOm)j zm33ZMVhnSI&{}|QZWLVU{U;{+@}`@ zkD9Yhy1#ftAF9*spJAvb?L&p!%)<{^tXncca03zPS_Qtnb8lecDqC&1aa6Q2m9167Y)c<6rh z=Iq)n77=LHDlpZu0&5peg3;d5!<6fwnV>4VY9Vq%BjhqfL`cj&hRL(>F-!_{OK5-f z#Tg{o?u1*^4M9ZLIeTRdzZNT0G*(5|Gks=J*Uss zJI(5zhTH&FeUkmM0MiRdi|f8ix`gVfuO(cj5A4nIDRL@9yMxcxnf|to;&RE@N~X>( zCFaDRzpZq<7lIh!W~{TR!j*u&xfdy$rH#=6aPEP9`GK_bo0NY8&%~?n?$J%d$#0(~ zcUT+HshoSTJJCyFaz9YI#7+BI3Ngrjem4lMu3S)H$+mT!(t2aVqgdv_NM5il#rKcl z(Kh&QU6})XSTh9m!Q!lHH`f@ke*zDc`|A?iz-FfJN$9e%XF$~#g4@Ab!D}7$-0?WBa_WeO707o z9h$kA$og&1jeLi$@_v}(k~IX_nVy&FsCUp?VsDU*aFkpL+fZRZrB8ZvZL`Xdx5LwU_$nmZs|1gl!uHMu~ z{Ln0fM?AhDvA(MUJ7y`X7(I3_C?I^{m$+bi`Y3iBgFlP{d2kCx}uk zUtoT3zx@A;-hqVtU+W#zS&sj2y`xx2?As1pDF59J~Ac-RaP+3363xeJZ9g5JI& zar|RML^aJpTUa&iHGKL_+c^$&Vpig3HVJh``69l&zAkw|d5)iKEu_!G|8#CSbO+}6RaQV9u6Fp58i`3G2OeZMvM+r|S?FXLqqO@-4-IPd+FJHMemJ(wG^%Pp)rVnouC`ba67nW_=EY7ScJ z#ns#xM(Ge#wfd z7L59-nRcNkJ6DTr%|Dro9r}+EN99zJ-!2LZc?OJD8&#Muf2GWay-ERwiEQ$%f;_ z+w*P_S5dVd>1C*FVEp={Nyw(bVX@hwCQS?2E_|_)2q#C$qdz9WQ3b$kgc5+nAMS zDovH*7$cF|B7KoEp&lNHCH9Ut2cJR59}r(zTi#pN7oJm;Ls>7uJKak4=-;*|p5ld# z!@w`4vNf(Z^p7-H&x5gOg)D|P_H-8CNiJA+uV95Kul+7ayYzOT3{v84;Sa@fiX!2A zs&z#QA)sN|Q!JOO5veN?bEbF2kubZGJD7o1*y`y8gbJ7Xa=0KM5>;wix8WK1`k2yT z_bes|?W9Zg8LjG7$*^4o`lsSZTmAS7*>E7=4<@o;q@lfZh( zi9vk9gz8D9fhrOEJSg5x_nDqkQ1E^R5oxpZJ5;FSG6!@7*D2^(4|7!l^Qi3!e|%aF zj4!=@uh{r>Xu~(8FjE$UTTm{eORz;Q*uX0iEsUcjSb36dj$MBmG@kmJ%EJxr z(*a8XqGw z_B_1$#%0R=taZ5CS2$T%uRPMc&U!`Aw=+`Dzw)+w%Xua4%lBJMM*U!IU>vP&n7g?dxfQl~G zNdSRZQ=cc~2b!qCxjTcTE}W5yr@6Xu-V>0;q;JcoC*aLC+LJ{?6aU9Lk`j0`Abd6Uh8Cnwt>`U{!Rt?L}2UnBYs{B_QjdLzbSoiO%H1$;cqSu>X(D1L^@j2{41fn z-Wn-dXk60I8Q`qt(UKt$ZXx$v6L$o3eHd`zC*m1YGAC+Og<_wbdg~fVKagJlD<3ww@RF?sQ0I)6N}NUa$orRNS_MO+{T9 zE6rh%JtbftXUlPyRA3aKTr(O+yMJ4xUxsDi$uXmm;kl0{i%0K$o!w`|lqMxh$tr78PCGNOK@KE>UNk!sAmER(z)g2yD#~+zvf?PVBaSK-PWnsShIg4r>N$#ie_;|5Z1jAME zvN@}D;XwM2f8G@O0=y&a-V@0g{bT1chQ4P5_k4E*mO~^!2zhrIDLZPr=w;BnP}3S5 z!^u&1V{nRy3*80^(Ppz;_}YzYF0$IL!<+ zn~>Sg1&cKLek}RRL?OF&Ws5|ud{Tn}Gyb(5M=+87jm)+wSY%29fAlY=)5~QC#Zapy z7qU_)5+kzos+_p6S#RI5brlDV1gZXwC=V3x#It_Wt#eUnt!}$!@3pW}D}Eug=jbYJ zh%=``x~$EL-!`VchMa?zp(or>Mixvwi5}!jBY6(}z-bO$O}x^hfMerJ3pLQjRghnl z;3xvaRDP^KLfx!w$DfQr-;^5cL={vU2fGJGmY zf6(EO94E)t^=&?kKDC@s>TRFGMPrDj~yCQ=J~QLG)CPVgjl9kbBlRFkrLF~nqiK%9JCozhCL zDK7q3qXR^9IN>uD=Ex+y8XfS7N+9ND`!?S&Yei&aCP=hTFWq$42l>7$g*y!P+l$rl z$4}hJ0Zl_|Kv>R}wo!ftCT`IrTtgv_HZe8mWe*w{yT3Pa%DDeUl|B)Xh7U_~koOr$7v_>{XWrU|m$?^V;#n@>Ty;xIq5H;q5%T;{u(mo$H1I-KZ zN5s6{Y~R3dNhT3Nos)%!g5PBYVf3;43wpa@^oe@AjpFy!`rF0Rts>c=jP2Qc?QNV| zJ~NVlF8WR~!7tFC>VqKYoJv)8VY}Gz=+puU8F0!^1|s?Dkt+$x`rVY-GmLXeD&pqN zdvjLGC8P$B)>mxWmvGjZ*ve$P5Uyt>3+I1U%isaI-{Ndsq+1wY6meNA>Ar1IZATM* z+vno;<;wP{ucH6#aMt@V{gH!JH^N&YXL-I)M!H}@BqX|V=$7nXMhCG0Kgd9OH98_I zfcqv9Axb|I%nznn8ewfDCu+nOL6!&s_0VQJ#(ja-Gd~! zuRp-PAoLj1MMJ582Cy5x2XBest=D`vVJ9TVCiZ~@I$l{J_mv`RABe2PEi;8&bw2m5 zOdqmYL+GVHVsu;|{=j!#7PkJj{HD2z(r!1G5;iGi&WB2#mO>q&268(j<432Zm&h60++9 z;T=*q%SpvRSm?0-@k-jg$L%mqgJt?yb$2=(JJZsy5_Ld?5-;=611n69O<*3@h&1Mi zi@*a=uZ@+vrO{`;Z*XY9q!2mn2VkKt4wgq7fUx!GnCrOfXTTQx)zXukE(8_}*WvzA zRL^Z4e;X?_(_YtLR9>xV0}GKMB@5gK;BaiwL?z*)&lmb3%=r#?r4+HICpQ4Pee&aT z*_-v%+3XaoBdoLDupnIUw)f9(?*j8Hp+usS9@V<5tg%%l`v{zB_ZfNG3JE45T7ihE z_qcRCV##lFk?CEsLZGlBR~Q!o8Ta=NO?x@xD!6LdUZw<@MS&ea7&*;a)EZ5(YRGK( z9qd?fh?C3ZPTn>zat4Vw$m_SH(*g8Z(Xo;o!9E&t(7D=FwNEY)CTf~@8sent0`=`} zPsf$=C7ugQMY*U_spblmPkBjVR`2}YU12Ke+LsQD~{KUOAu1lO-#_*HgL3 zlO2(?$y*@OkzNtbA$~Ga>Z>|5fKQC8H9${{>#J&F@|7CYnGKSXG}LhUbb6c^7iCG} zk&TiBNlorNFp%@(7fBOk=7fLA?^!lS%nnm$y|z_^HQg1~B8+Ac>|iR)LZ1$uuDbAZ|;}ep3Wfn= zhG)3mc0_y2Cs-YAS@-U$Lq)gcNdURCfqQrPQN_}V`gm6wY4~! zmAY~WNmvkfZ@M}?tTq!t=)iH`qI6y;zm?asOhoapw)|P|;UvY4EFG?}@J*RE zctaA|W5VOrP1S)>|1&pY4`E6MBDye54HPT7-A}5b;ZUlli;KxxqE;Sv`ny!OE$6hd zILRCK(Tl;>y9YhiM23s*PV+0Hv+|U!i*5b-1jD=32hrT|fFkivbFDc zVQyr3R8em|NM&qo0POwib{n~|FbwB!bQM_CWXpEcO;QKj!({SpjqFMM#P;Zu?8&=V zGFt<7gCs`HMo$ARnX$FjzK!SadujVh-i5-Ufj;mgN|tBTKWwoZC=^arg+d*2oJJFp z2UC_(!E&08yT=opi*Sag$?rDv^m@JC(}M%}zt`(k|KHy`c>25kL2rNW>C?mgy~E%2 z`iH%K|97ajq4wuK1s6E~UGK(iRXg{EJUC66z=E=r54I4(xu7E)iAgaGDeERQz2F1% zX$v8Q0+iw@86a36=!_M4MBoELMV<`MM2L(Jy4^8_>JgiEvz$$dn2>@85t~knl!{q* zLXv3^Ng&*V`#$3$CwPiZa`rQc#PYCNMsh4z-n}S>Bq%@BJ7h`@P)Fni>1-hsVSy7i z4kIi?LS!La2o}(k3!EeaRNo;%l*ivpv4|#()gvO?q$IX=8;ix1X9=DKs`GCw^v1%n z5YKQ#21vbh(cv>L$h5Iuma|J56CR8&^8qUBgb~XL<8mOwa=f~i{YEG`NeJddqgo%3 zmP)@AgbN--SusEdhrM1iiniR8Oj$k~po3no_l7o>EvQ^a8B0kj-0p1QjQ&7!PFXrY zm;EiAW%g$;+H4vOOoRRK9j%I zk|cQV+K#OitUHTZl$- zz)a;FUxf-l!ATymRFG6uU^##4vU2LCSoWfeIVZv`a-Og;3!>rS;c$2u?H@c1vvj<5 zL1tGhkNIFLKnAs~06BOCsETReE{G|nslYTPIlxdW_mU@+6)DB!(!Lu~KE-)-u@xXP z#WWe9F~wnX!m9==pg!s?=>2T(igx5J8 z4`*f3Da{#A@MSo`@xLdOTsE}u{|Gvsq$?RmSHEb4zg2%LC1;Y#{>B3&<;pGO z;5Gn3c3aBL>6jn^V>pt-ASDg;iXbcDeSb;vTnY#*hnMB=5o(~ITb_&zEV?GIIt;l@ zdJOqx1fVjOivjw&_jPXzAv(olqVUg2#<;9Av;3kvrFqVB5(hEONhCf@aoXPtE}rwC zK1Kna#!n9bl)&Qwy6lC|!k%ifNRktl&}cS5uSRc~ILQeosjwfbzwyKai*px+p@<(dy*~$_4M#tDHp>5*Nw5I&wfw zu_Xtx-!PEefM_4Ty5YU~caZWuecv}f((wm)y&K3A7tiYQ*(ndwj0{{GLb$c8AgQhXN;x6ldlrl-yqax2qDw3I`>wU!1WcGODr2I-X zACM1n!j%AVTBLkw0YU>fHvkZp;?WCCrYt=p0^pH^5{_63jGoBOlQaT`j?ooOIpQS0 zB)NQ{&p(nq_QT#EWhHV+@|hhcx!QfnX>vQiTW<&vF(K&XJg4fg+ru ztQaPgPt;evLjWi&dxPl-i?=lJml}4%7LzDk?DIQkychfY42EqzLbuNYHR@jyq1)aD z&Ct<{H+MhWvfX&PrmXYqHf5cKq`BZkE)^P)qbOoUDvom!lT=WgaIYXuIf;s#oL$iD z2a?m#Y)z3sX}bof^BIY&p0O2$sV4q-Q(@|1g62)oZ%ta$-CkB=d)_48FC(#SK(fCN zt3(p@3$ph)4z_z=`pMSL-yAu2{w%i6Lrh?>`I0@0K5lWdroQU1(HFB)42e@AcQ7+d^ z9BP?fnl1)ey^4&T(J*Hf+A4ktK(*tC2$AM@ia>aM?*4Vxol)ixLh|LbOq;B zfuZS`570k<)cp?-j?~ru1tFOol$?yohXLA=3+en(42N<~#HW~K37btxDn4Z_{$%Ip zlU#eBxWIypii8vKIQ-*|g{HXXYuUVP`}|ziJ3s$XcNP8q#$C)?I#+9XN(4SHx5s(L z;&X{Rp!8#jG0CvrB!8+7%6~r!pFIA`zEvv?nowI%LUQV43697Vh{{*}a@FZk*Wb6c zwwp{48ZL8Av=sz~&4Fl&003g6=G!OOY(L3G{u6>R;u6Z+@87)~$kppzrckafvv%S2 zwBQ02S&W6cXsKUKI2JHrktdQaxEC)2)T!U8C~9NxW+dQpQI#A1){*qCj0ULFcE8i9 zjQYiMZgov^nHGYTb}oocK@X&?RuK6BYgciTcd$;hbxqlDKkPpZpBhC$OQB8`3z&Pk zIf@|{R1|{1Jj)Rn6-lDTQLc%cXoCdVg8WpOAO%=$5O3SzB=IXpY&vDB*-9U;x3&?F zrW4F31B8cBe{X+l8%<|=BVc@SF^v^xf!iia50%}I#MUv)k%()19od#Hdn$`c;_F>l zY11J^lyO3Gq2*$S{VJAfe~w8?a!Fq8n~Ue}`4~=C8C80%w)aPum?n5g6DnqHWnShv zYXxwYuxg_T5tTP375Hl%Y47`67Ly~{eVUHv7CI#n%j0?}MFNk89EelXr!1=?tUjEY zh0*x+Ot2h}N%J?@RFzMqWM6qN`NE-uU04mS(bZYNI zf%%>m>5#DtROAWDax$VHV99!m7*-+_d*5PeS@zqt6LH_t4>UFR=V0fhA%@dA|V6R37>RyXY}@N>OfNNXDE79nzc5Tcg=bcc8I`vg*(BBSPdumV?lbo;U?0vg3peaWYUwmm<-^ zsA3)3K;)E+DHr)nYeVGhM zGoBq)HipIREF%#e%}~Z-clVS9PF+Qme{zdi#JdqoBa#W;&0x~I8H)o@=29^W^t)}G zWjLR*e8XTliR7Vp(@NPLnrx$lAQ4baY$oziFd=Y)Ac33;dBg_e-mZig0JtE_=_M8<$Tgwrn&UTo zK5^}c;ApjN32R3Kkgyc$ofeBTS5B*GUmXeEAaD3a*R9wE_TWlS?cC7IL70CAGW=TgW^E*-(dOac<*gBYNlj~}i1 zFzNuXP|KX5EYU$By1w3VE8>ivgN^K;6=qjtIAQD}DDotjkevL*6t-r`PM^S@%cI{4 z_3o2UOtS23DkW((l+c%R$vs&i^dva*+cvq)Gut|?jdR-Ho7hiwM$;lmj7R`j zIJ$Z8X&Y>n2L|Z1n$v&CZ!sI&uZ=LhHQ_9XvV=Fn%M7oYh!YI6^Ccx}prIzwbR5aZ!Y0h2g4kg^F$rXimwA{LWNlCX@X<8yt%yA4iYftZj$WJxe8 zQc%qpmUJ$$Pz_5{cD=W83NA)5JZ); zf?Cq6vWna32XF<0=y%i}NHsA#*U~nJeK0_u0=xGcog91))K9z@Br4a|( zh9>xuAd%DYnB-EHLGB2KK2CIm#sWP`NQ%P0%Gydv6#f7*?PJt)d-eoMXg$*4i9_fl zs2nu&9+Wed3lg7TnsXNpLjU_e|1Wy-@goZ50SR4S51yc(=`KM0biIC+2i+C%*{O)um!hE;B?F`mfTZL8umvenQ}0L0N!D@Br))^)P>(lJnj zhGEbXU>T4%7XLhY{qpo33ZVB5C7s0v7$v-^ezjEdd}#3CC``r7gL9@|n%S?H{b_|J zjH9pHf>!Pfo*=ic4JGG*cT_qdFQ!wRS9+n|>BY35^pfpN3Fmn1kNO++uA!ZV(o44E zcy)Viy>Ps}>+QAFTCz>8+Li*P*QU|6zdwCFc!K_1ZPc&!X}~j@hB}WMx@#$Fm3|&so%RAEddGxw zxmRkC+^d1*M^SHu4(jN|+Cd#>8Ff&{ncuYs)z!X!Nh9mfd~Jf#c0IO?TSV zP^wHjJX4$&I7wz!0Yp>ZtJK9rBo*rtfTdKuZf^p8-yDcUw`)h&{;UkG7H`eiHtJBl zZmG6QZJzu!Fa|YeYOHHVGzHU)!H}RRAvh&5Dl!BCH&Bu4_$~iG4a#^fa{}HoupZHH zW^Ym;FbBCLDICyH!Z_F+*n2;Jb}EhNHoi}jnNoIp*iv5J6yh}Y2duVw3QoZCPm&(! z-#4YoOkZfp&KBAC>HE|Msd4s~vR4C*wn9cQq#9T-kUJbDbVQh6Ee3fYI zZ}8Z-1)Ceiz3?!6rr)&QD8hGh>2b_b&?A}Viu8!TW$7tn!n==y5AQiqH>SN-;?=mu zVH?VIgHubc;Pekl*(uKCtPmVc{DI90D2a_$TS50mSARJn`Kc~zNPs4V_R>SOS{7_o z#8&&%LXW?rl0(T;wA>hSDM?ZQ61VIzbS889KrNLv5I%maV--puiddrS>)_)@^pyti zwwMk<#tmERy#PXPn31M#aFcH>9l)5X0jqK{;I#Qu;5Yk|0fn=Sq_NsX;JpXoc@!B` z&De{D?QNtHDuJ8ZuQ{k#aH_IrJ2{^%Ez4arorI~<`YdCt1dK<;tAcBy>K#`r!t1gi zk?=Ez-8g+cQ?`mE)?A=@Rq4$XqGO!e(G_e=#DwHZfRp`kd<!Z1w#q= zxFXO$mTT06me{v)k8?i^7nBHW^AY(Va(ryA0imKdla5ui3l7kdg!MzlI04^TIY%jr zjr`+qG&(Ya5sFzxBvadY%VHvd?6|F^`sD!a_rC7g#$Hn{f$}C|mXU$6KzVht1Uob)$;866TgfrIZsdxUCA_>DPB=ru1SY2*llXB1G$$`PvJwrW9Gy{31= zgbj^(?h7|UWfx~OoW|R3v)nYhg=*-O8%E$X#(9j`@MjW<|NWo;%jLquF%~#PZ`9O{ z*aDm4ZW&C&MG@Sn=&%qh*Y}-WPR9YxqX`ux5=BlV;z7hxd2CmT*ueD|%PTl_vr*Mjs3Zl8PP#UvmYl%=SHYCQ zv-bKrdKN-_f<5U>(?!`2C4z$$sX342gl8;OmS0XdQfx%qnx&Yp`>=D*lu4gyNtZ$y z&dak89jCH0=n+k$qyS@T@4oXfOoyz^QeqAa7LW4$AIK-$(cTrn}kDKP&i5h}-lMJylC(inB^*BMYUWzxudIxjj{?06I>nTvKGt&RbiL znkv>{0+-ajDlFwfiZr2#qX>*_$5W&3zz}hZ|5Bsl4UDl5JM;9T5WA741ifZs^c|Qt z=+f^wK?xg=rUE_ElFDPma`ZE2sS~v^VdJr~iIL*jSb-SmnvPsJMv0`qji#PMVuwph zz8>ZY8^4xy2fzu+Jxc#g%v6#O8HwQBFJJ0&woibxTLp|`>0YZy1SoBl=3$C26fJTf znHKXR1+9m*yC^^y z@9*^{JIWaVbSC8$LQcj-f^+cCAe=i?_Mg4r>mUF4Xv#nFPt#A6$LP_NCJE&vVreXo zmFoW~O$)gJCalQ$kY6Qcn#~!%x+R_S)ZFI?!VoHt^OsDTO^C|Bycqz{yr}}Y%IaeyGvG(UH z%it&?oWEf)99}x71jql#sUY8{5$XKs6twOtZVL#Ri0A%W2ld0=U#TOOYFBR%H0(j) z9V}aQ%17@{U(cBrJB3!~fQmF$`$^r3q(t@z=OBEmTOT2JTAq_gNa(z&EFIOZ1=MjV zza%jl><9W`}-`>4D zK@*(DiTnkd&tP0|c|h5f_^L?huE7l4U+AMc-6eyo<<(h-5;0 z+a*L01KO9Iv7{qakxlb4#fIO?jb#GRsb)>UlNgm}z=gK=j=~H=N~nz_sYq2Da^`I@ zAE(mg%+8mb|AbDd_6ksDOukIC0&WqTW(Byd=!UvJ`SwT%(hZkJx+ZJFp-!5h9>}@S z@2Q*%b(-T=>^D|)uExRPOV9w1fJMwy1VQJr)P|!`dtM%HL(GV>wsqK;DMRA| z=QtGvxS_E$@KDnzO>iF1L5XtoL_1u>Ya!IsW=E0lm~x=pS#3IuVR6bTAC&EaA^xS5 zRz1odrGg&mO4Wm;53#hkkaR?CQo%Bou<`8gWai#fc@>?0Q4Z^;bUQ(U?J6a(B5}KD zglSUbB+zym4NhwRW+|D|iFYMR1LW5)8p#Ujh{g(H89zz4~DJoeI z+_18Ehr6MNmZr%q6D1{j5n#y9Vt{(H<%T(O^^h<|oeMJS*lyHg7MU+F5ChbK|I62n z(UKfRhilP*CM=1$9DqO_*Nnvj!a`}%h~ix;1O~Ehc5*L+3ew-Ev2&nJS*m0?tx6IT zRL~eW41gTd8140Xz3}%>F8kpLiz~Q!(%uc70<9lzq+oX|`jS>F53uYW0G{saWsE=6 z+mQ#to{!;z2%=}f3A91apanmBm4p(;vLTKxb}jX$0>z9d33^DSP&&qv)EGesy*jbe z^OgydM}R|IWNTmGwfg?mE`XCC#*A~CIWD19ZF*AnqAH!)%PX#2;5nk{&qh^U8ZX)z zat;fURXX8{Wocy;g9|Z{=Te1=q-c34a7;H{+&1-FJ1`&gyG^3r7J(|MUD!xVB7 z$3RR8^|~oJMpNV>W6z+oZtX}(jDq=jRsU>bDrw|t3Kmt4pmRxbt;?q=)t*0E|K)^t_v{=e9N!4l$?E92=K1v6+p8f!?r~j%H4PD)1yP z^OTr{QwEFT6Lysv$sX*VOyaZ*j}4N#+WCY{$+@k2o?$T=pstu^-4YpV?x!mSvQqdn zXQ^Bw<#ptl$gv<}Ef1+X;!~n7gME+s8qz#86RJPGuwP!*JnbEVfLK-wDpOW%0%v3! z_kiOC6I5}}YW0A0P$e(bv;W0zn-cs#CdvXzPL{e27mW=>tFuNO#M|yk1W*?browsFo9kHS8;)JyIz%(}03m6ELIVlba zO2|l{A(=@2%2I<+)s9z(YSRJfpv`i;t*@GGb=O|B&1|oYgFQ4dX>0^UbfuH3_Z-x+ zW{b}?_{H&*rYD&5D~S68*M>GXP}LsaHJUJ+)R?DMCd|{!HJI@?1X-#;^mtlxSeeQZ zLgN|&6CWTG#j>fWNO*vwX_;@uN@F$#+>l6CS_%OTXtlTl4>&-GQ2$k!anw7la z8;?Y%D@$9;@cf3Q9bzIXOheY)=j98;CGg3mg=Y zTtL8IRccGEa=70;QCpRW$jZl@CtY2{-gLLY$O(?WPm`G!@TUw?;~YH3wY9*}Bb{&0 zgh3El@^}gP7kCNfLd(2B@Hk~DO~>7B2mY3ONRh*PW`?&iK5i6O*za4$5-O5B37JUT z@*N1PM^ZkvaX;L*4vWJ;)4Z?hs2<~p=`SRvLdB3VGcMms8B*dq*ecaZBH<0$B({}c z{VG1CJRuI8uy_)(r(QZYiEKe$=&LVg)_PAO5xtSB#V)Lj#SX=@`V~LzyJ7QY&jM@7$4>Ydw}Sx+#`aHTON!WN^r!N3WdH0! zMfGne38DU?h8T7Hcs~%6(!|C%H^JUiYy#`=%0qQ03LwIAdgP{SDJ1Y1U0<76^m3&- zU9(eAS*yrK7prV4q(Zd{7O`Z2-W{Llx5^knvl9#6hanzW+$?E9~^@>9N7OZ5lnjFVT)&;aUamw9bz>1uw6pc;x~nu!@C)YC#$ zbs@SPE&1uZkyC3|^IBVw3Nm%>MYZC-wi*#|_LvON!TxjSos!9&;wu=ZYwRBE($Rxm z`u(v>2ToKmJn+p9jP1alV>WNqNs8Z>9_kZW6x55gJMcPKU$T}B(S z|Cux`2|sp1vZZG*F$-JzSXcq9gUBMD#hr*!2$S6eQsj;GaKq|*nB&N zhbr8oC5hh7)^Gx`7?84)OZ+ z-cuKG1<;ods(5#GUaD?{)vTx+M%|Sim+qBUyLtkb1)b$=Xs#~j0#*f=1#1}J5`axn zN2#}szBAP{h3|*`=eu?A-OD}%^zNe^xHO#fYo)wql&Ifn2W8_ByzyuR?CzxqMALM< znM)^HHu0`Hye%ih*rUSvqkrPy-{(L6@$sLQ7Fyz*Ws9&`84aD9<6K19dBO3RoC_v! z^2xj}V*foDtegX(`xIrxdBSjfjxR|ri}ZW^@oL@K8z=Fe>h-%omm)=;yDUe&!Fm!6bN`uc>joGs`q7a2{JGm$a{RX#r^TE@cQhj^GYs zSKETm(5n-Us8vaiSONAvWiAz^)o@iT{a)BV=!Xa4-d^p^e(lXciK+4^YwTcX8&XEP z$~;})-<4Ni;Nhgb+?%iM%B+zDRX$R4CS@Hfu^!-9M}+skSehDAL7q9zdRI*x)f_kGl`_=V6OHY!ST; z8n=kv7ec2EXd)pbckj*do~>6VPZlgAkj>KT$`*QW4$^@y4mn@>r&vEWLz% zM*=V;^Z#hagHQNkt&fBu=)&I=6weSv;B$c}lTn0o!v8F-$Gze3P&U zClkhn`558vXhO8Pb=hAVQmu-let~U$l8dz|+!N;Gu^Zc;4XX(IjVi+a z(kjAo;O_wyYEulBvwM&9fw!O#e9#C!5ZYTNw91!56WJ5v+_IR!+HiUIfw(r)!h^a1 zUcOzdFHs|NPgEw2WKhk)y&{1RgyrUh<+FBfu23eSncHhCn8}w)vor77;6dT|g|Esl zhut;ipHXt79MwQCVCzh4REx&p?rvSP#xvkWfyx^hCS11=%7?}Pt5hKDScwp z@4BZ=)w8CpMt@({*eZkIK*US(L2^ivD`ssMr1l$yGO)SjA1`&;`o>lw!F#Tv%&0Iy zP84vOx}M(hV2hNsd6jv^K`_|A3>DZPs*c!vXp-kIZnA;3y;sk zKQ6t!J+tTsGS@=p+BTg>?Ai*=BYAey+KqS=s~Ffj9*i9oQ$t7Jse~vnj<448j+^u` zIv^z&6H4MyTH=kXfml2tv?LuK7RKTD$a<#-CNN7r ze|I0nZ{m<$W`|79lNKQYZl{L`r5BofJ|NiilO8IJg;=`RH- z$R{cPCwQiimH&+Rr;H1hlTUdToo6&7BmT+g_~oB7K7w5skr9V=*<{-B?CYnsxWaa(>H!)gR zV@~j23Y?Q73&s=1MUkc?(bZJ`pniB5?&&useUSRqnnV|Q(a6i>w@2}mLXco|MTQf` zE)XxW%mfs1!tubSF2kv-*jzMi3i^Y3;eOAk4NSyoCq*+&b#4MR>MEUy4sZj_fbZ56 zUufG@nxiW`Yq}0Y8H4;75D80zHwLX`+zkK+^R0Au*!1}ys>^yg>%i4!_`bt5X_-l- zP|@#q%ajY)J)*lBJso5&vpGY|V0*WAy!0_PAY%v&WR(Nor#dKJk&Sa46OJCS98oX1 zDMbI{kJT6ulgrW{5aabY)>~Ew4I%-| zEst2Hgnp2il~`g_)J+@N=E^W$&ghYfPm@qC9;+yvWeA%>C-Na5qNL#TQs=pdNjfX> z2u;d7yW()FqJ`*G6@I|{GB=qULDm&^xD#hRR}q?=xN(kaXBqTrU`57?6R|&H5Pf-# zQF|nN><$q`7Ig=~nIRLu2uG2WVj8nF6`X_(Sp-*(yY4cY9hSf(EixGnKsTg%g4r;vkia%}Wyd>ck(Vra`J@g0+dkR?kN4 zuXi1<3*w=x@!c(>b?qXz-Llz+pk4@lY1y2;*v24^nxj?&F|h83w?^{4?dIfr(109z1rJ@4d?u=nHflq*O@g!k zuo{~(MAcgQpOvpKpa#&kQ{%I1lHWnX2NnC>3ACaGJdPgmBAOt~(a}H7(96G{Jyxd( zo#N^E?W?jxgcBOE$GgzZkVII5*Pe$#mf)1wmWDHc7lEkhBvR356v;Psr1)_B>cy$b zDp*yFlqvMIG-if$!nk;=WHg^$u9_F`rdjdSxu_u{9)x(c$3s60A>1oLmZT;Qaq~n= zfK)xnQZ2>Oc1wb#?WXif+jEjDEtwUpL2Y=W{x@UqClKuIaPNy`UNjW#FwC{IFr>=Jr z*Zae&YU`>buOqLM%}X*X_hi`_EB#wwY9vlIsAaoSpPC9z^JmDMR7I6iXvA&LIE_gV z;T#>GyjQ%`p96#Uv7483MbL_+&V&BOSQ#8@o~s(b=sQZ17#Y$4nXG6cNRF-~Nq{tv z;M!2i%!ig7*9f)P)k*`TbR@PM{{R>u+ zZCW#iyqNHA4v3|~!b4Ph17_98@U@WeU#f;?2?gVdD#m6dqchfZ<=pD(x%D)2=DfOm zD1Et1L7J#$o^4$0usw`x8yt*?ZOrB@aSneC$%iYuo+eweS~~H3xa~pt)-SQWEV&oHz}eJD}?-0-RmX1PzFD$&?`6D*%CXtff<@* zsXYe1)=3a_W-xBkXU)w9=4JaMD z6Is$zIxWf&=3NioQJOHn$Mds<%^e+jvq^TRbB#oA@uEy`PST&HlD)OhF$xe%2~4la zVK}w+A7wLV8HvK26@r}PWJEvQY-u<=?UoiseQ7wStZEXu+5#!H63#$ZvdJwFAyPZ5 zkhla0=M-naHX-*}O?L1vVBNG%RVHSKN3RyE5>PF!A$7DY-|uLO6O~t38xok*_N74N z3M14mWCP*j$2wM_1frG^y1ouReneks0B?)wkmTt4I&7`?0+u_vzV>#AJlF#~bI;O* zRXGt8d`Zwxf#2*;1{BURk~%pZNgCU&rJ8ZCPc%X$a3f>QLB)bomHi{pMo98m89u?7 zm>G++s{3eVL$;y2?&cH;KXbNW$4xD-glu!bZ(dV+8-?iD$^9ePSmkL|N_UbA8*b1i z2&(Doj%eHyEh~N=Zk6?JR(wOdVK#TY+v@s{%q+8#3{=l8UVgrYjKLss+PE7TXz|vP zH?1aF!RTfz4s3dS{jS|H1q`uBe2g4D(LdZBX)dk( zH3>`cCz}@LC$(f0(@-gR6&3PLxOXXzhf69g=ifQ7sJ2Zi(zeZ z$q|NIGr4Yk5p?s`A&rZ+ti(;Dp1NP6QcrfBVa`zyaKUmg#^>=coZ=6`Fk#U}5bGWM z&E5og7UE@gMV#aW$1@a5iQMkocW_Q|XIopH9X7`dPP9}@%)W~U+6fxmwRHTuTA*7UGEEYhQ z1B``EAu0%UPOxUR>Qx9v3nq0|<5F{$x{)cIxK#-jgp2Mk1<7YL9m}=U-IjYNfGz}b z<#?0jb-y||aP{UVmG*^zrM+7q$-S3|7un0Wy|Dkh=GtP%XE_^&+puCNCEuDQ1CZT{ zF@rwlHrX@iPc@SU{l3krL6ILv@dLa}tvQ^4Z-zM|S)YbGp-c`+HX=yn1o=YClD@p9 z>=;>^l1P{Xp$pB6Op@VTsB;5k0RSt1j|$3)G#$UJ1SLUEI_pk3%#Nj@^Hyp0%goA6 zvDCKp@vwSat%+dfZB+_$Yv7FjU=l}P?v->%Uq@-hfnzO;)0s;-5X>}c!V~&X^CMgrdp4dv^#0XCC{-xVS?V##exLW^-E0*pE z)#Z&hf1D7US83bwy=}Pst~FimqucKM4Y6q)3%mHHl8}weqki1!S7) zodlvTa6vh?iQH_X1dla$G#pX35%%q)u%8bFX0vSdNC2l0IJR zI5)nRe^|YTE0U5kDFGEL=V+?}$lU8CZ7QK}=C!gOwy@~}M?=hkqHFtDvjZn2nW9qb zwQC6gY&d8eC!9HVhW_k3zW6=~v14R9@|Gs=IpKdJ$@B+hqFdoh)n(PNHk2-^dBJ_W zw(Ct&i3;Ow!>SaLrOcsps+!o8u*$*C{rCeOxm2=|tI=2Q^?*yEnT=3KYiAW`2D%@Hudi}?(K&4J=GfbX>(5&?%1kuduuzk8NWq)m)8(%+oxusaW++o zcC*M@8eX*A94lmGVJ^vCzZCxG56vxc!pt70?(O!p(gEuCVq5tQ{y?Y2 z6mj}*&@9cXc}D+DngRw#Y-(f;qnx_RsUU$0!jrJEzQ;OCahA=zW78E*j?3ybUH%-+ zQPYkFvK;@nzF<2t__TCj-!}xnpXr@{cV6;>*O&?$R$at z7B$04+)5Wv=~7SA5%XXvFa8yQ)jj-?V;yA_b2%AP$maqUeS8-pJm(Y~wH(!$`fPW` zdrn40Vkq(lO0-E;PGG{(`_tF{;oq?r9JsbgCM2Iaz5nn3{9i6t9)=iF=#9GN9Z&LjzM6(o#iaUDzbImT{Cm^40cCrsE+DAsZchE4&ET3+sNV0H-{l0oC+FPAJPb? zHjWv{cs88@=r6MbN4|=oTgx)Nh=qj+gU_1n2P_lqg3}j0@{p z2sO3Y@qTr14d>;w6>zXm`n&@+ZwmKdXfkR*2`)`(38JXLOcO^E&Id%L}{*H|fQ0(sp~~WXc3N&!rr|GZM|U1iw2ar$S3&Wi_4Z)hWTT zcOzTmi9%)2?W!XEuB_M8OT`24T{i)(NdML2Pi4YVeS<=Ci|E z#b<}TX@iCmohc(Q(EY<;keq`_k3g=NaeIrPyl~@PW05xXl{zJgt|`DJONuEuPB5Lm zBhxIwf;d!wmzHm86f_hka@ANl!MTN0M!`2M2IJq3+-ET7-=`7j{OFdf%%Y_*{-l(I z$Fx5{hrPd22hXOnQW*(P*%%*db4##M=5D@1$F$nD^HNsEel*n37%cm*O`A9;HHU&q zIMp(W>TuQ;tuaLZhYhP0;i?XN8DuSaUa+m2-q7-j4D9a^FXRyc;!Z(&st~=Ej6GRIt<{?@NSGA_Sf%o>GeXU{=LF`q?8)_}o zOLsRINpow>QqV*Fy$rDA;^awQ&3o75Vn4ItxiX@Hz-*A?)KCP+=9$NOK~_X7bqP>< zF*l?UwI);=f(szR=OZwkLhGamwNOYdG@Lclgc?W}WM&Y$L9S3E&cd|U$E~!VhXPZa zIT&bz@w(MHSez?}kHSAbcAzQhf1Z>U>pBjDn<1h(1+@C&nWHu5dt=YJ3W^y42N z4<7w1c5tZLtbgoZ51g#J%}6$O&^M_^&7KAp9lgai*UW;k@z!}cDB+X2Ls6<_K~;D@ zXJHtK4!)F~n?v3weUdy^C&}|+rE8R(=VVM-`l;ay<4tdI9!;9wn{(m0m}SefD^IYG z8wM15>Lm%0hwxHTXcP~W0ALLJOWQ1jk>pfK>0m|^2xsnxI_Jtt%AOite#n#_+uR!G zuBneA){+L>&AW|`RWLa}^g}Z>okwYv8Iv1zyv2!5*|FMR@<3__%!&D1UgRiZr+OZG>Wcz2m8MZJt5HKQdCn!ph-WlS@r8<8ZS*#m%E(porE*kv z5tz}F@hr6Uq(G$0%K_Md5!k7W-@HDy2&P~=tPYWHO#$sHYJdnHW$mKn-oY* ztINox?PIa7Sc>PZY)^QLSjvl}Q8){Iw_BB_uN zVshzm#PPY~>$bDI<3iI0w^bFJz9nF&?TF#pGupKqXwbHr_v2lAQ;xF%;M65rg6U=V$ebb%0tzy5QF8YfHw*Tqg$W` z{*I^wb!m4%EhqiF47;_QyJ=m7JM&|Dq6tf4xV#J0U9R<@$3opJ>ZD;homiQ^$9M?% z_bK|qjH|1r6y=NwwR%7fD`*V5N0o<+#%QnC>xET4%q?h#Jw;$sSW`cWsYJ_tT@PdG z1v2fI#YY|p)3e{7_*rVyolD8mPne&Ctu;Wt9%%^Pbx=iE8d+5PRfX-^4G zO)^t<1*KKQA3wddj;<%mdGS?Pd2d}hc{fJlGYR3!Kzuq3I24ZR5&KGP2k6t5kyp}c z{3Dfj+zb-784PuCZwLbvjGZ4U6oTa1e{zdN&W|ntQ2&N<``K@5LQ&`AMiM!K4q;3HJX1OjXNH(27<9QMt)Xh3xhG>tlzdt`M@ zma$H3+VH|D1Iil?r2YY*^OnJv>^*dvIDF#%+= zDpd9XQx7)N?6CL@kI8`+aGJhy-<+D?t{Y>=MsT;z%*Z=t8*E4HyG+Pcp$f{@FzeAe^s9$yHVO5LIRV0R!Nn$;QH^z1J}=jbPJNYSoX2E1q*)pq3(SAi%4 zjNXR3%_+;DRvdYrB_>i)OSZ;rKp>C`{YHSWvtZ`fcMwTnjk%UJ{O)(r58Qhx{nI zTe*-0WBDCC{A5L_jLmqy)ddVV@lM_lDNK5x&(PuaE?m?K zi?Zc}HV%9u@e!D*W>_ruM7r@hpOTvENO3|h1(~B~{G&Lk`*V?ZkZE)Rj{%eGjH)B% zyYR#Nc(GZ6lM_r&!X!E_Fm5w1w)vyqYUIYl%{I0qZ}~;lGBqc!;oY*yV^p{Go2KCb z$`}54fv?7e#FTc*OkU$-+VEb@bl0@7N#QazcC@Y`cq6=0jVta|z(*6#+C*=G3S-%d zpTr|ql1Kw%Tq8DAw>};MO-nxW+s6yNF`BIVY?44GPnjUoWZaS#D|eT zQ&l{+d-RFyXB~v~|J;yg;9{1SAw4ZExUO(Mk*~WyzqTeC6ueQ`tqNAoAF~PYdVyVf zHE)1xd*Hw8H|VaLvNflQj2~ofw zg7BYJ!7Qd4j%eZGXKDeFAbu4T5Q>p!iOV6%$js^-db~^{8ALUGrLLoGG(ti*uKtJY zsw!1#E|8Mf_rn^NBPt``)WHiNfhQe2!`l&&)AkPZ7K)ENf4gA{c>oYU~ z@^ha)6FdQRc{WZ}s{E>A2*K%8rW8M*S5_2aqbr|$Gi!HvQWeo2z@su*NDyHHF;x*I&S)chfz-NhAINc9#VK3hZX{MlbN z{JAckgEE6gO%$g^LEERnbeXu_px?xLu#0OhFQ5Jj_nMXYM4g1dbI&a@y_r^2C|)jd zpeRG)&()j6b(~W8;Rx~@%fu^&upnBf)M(?>TyIYCO1XHz|frqsr!fY}3K zAnZqrCcVUT7dG)5-HS%|4@Er73~9TR+Rf^FOU`%V9&=v?8oZ%}W-hO7=rz;-j7?Z< ze>BN!)$*&s#-PVrk<^;B&ufhhIXL7+_*5sN*=*^TXLUU(v19*47jP>BEyHj{wFb7@ zgBUkV6)X+pGV7$M2++8qt!@0x|cUWsWQhYLv z(KoMomhm-VbFn)H4_`8cSb!G6P61qi*J8q}$9}!pbC#%R>zd$L{%`Mv=II@_H)vTS z*P%JhxW3Csw1r9SEGM-PlW-$$VHAFRBy(8V2;_BeTWL&1KF6~rbWwbjG}<1CpV6G* z<6ao(@K}pck?2_Y@wJ^MQ2NBxe(3F46-g%f4OYZS_4I+}uH-P>i%ASP1vTtL4mA?B zQRY9ZDq4=H8{SWRSNUU=I~M1Y?Qy__8Tk|fm3K<2{-%@OUXq_1v)-S^%R%RM6V3&N zYKnEIjW?*X5Q7&EZC#;LS&gALKiV#?OT<1&al`6GiaiA0rryeQx?PIm%9-@#jXIDj z0Q4Ry6G^a@eAF8XquJ)u?$nN-PjxAN?5GVAjN(lxITYMQYKDk`gwdaKctsE~p`+wpeBq{~5co+MQ9FszOUv z28$$%2vOzkP-#5=5}(^laWv1^U=B7Yc&>9eKKh)c4Aw2+0t0W1UOdd@$e#L`_i5$_ z5nY&A!xUJmnB-O5S?<$LctPmf@XS(~Gn1|!$-BsUD|0_b|3kUB$xl<#Oa7#iw86t4 zTj-hhtHq#>%Y7YNO~D&Kb7;w{gDId$i0B3US|zv4H6>t8AkvZ=>>-O_2JV%y(LMuDjn9@8#wL^-Zi-CNjh61@*R>c_C+h?6NGlol?sD1=WXmBr zS6*1X@FnT%tZ#7qJm}EEp>k4y_N@I{wP03e*ArEBU1# z6MH6R7LJcg;=I|XXN%2DjhOdQKQpGnM&Oe(lQ>;$85!tw#?sO}F00wurG>4#eshB1 zI!UPP!#r`BAg+3y&KCSujGHeO{Z;BTpgpVwrOEd{hC-FCg!*b?uJrz@BQ(mUJzh-Z zI5`G}Wx;|~FjeIYa{u1^e6FP1wY`tGf$dh4@2Jx^&ab$8kk36{Ge|Vw*Y~agGvwzU zx7*2r7_lTTpRS|hG@2BtpLLmxC>7u4NmcOa!{9SoeL!N38*!8I_KN~e$05SePw(IzbR^p z#YFVB4DG^#z1_O#jH9_`_o?j@cl5HV8I)7xYs=xoc+{;|)ohdRwj}nXzZZv6my{ft z`{)q|%8Izv6Lgcc65)?>YWG4}2wB;CIdYFF4z$}N_%fs2xQSz?q7~JldBKmbHT@; z)d{GfmrcbgI_2liM~ClqF`CfkgBkBb&(z3xhwi*f;y7yj<%edrt37sJeWRML1-B+w z|4~lhh~qnYxNNc{PVF8ZF3dWwO4imf+0r+PzHpjK!TYzjAtzUwpyLbFSdaTJiH$(k zpJ}YNM?m%0Nov{!&H9R*A1Vk?+BfJ`_9trJtNdg+0vE3;&%v;iQZ^yr z_)?8dnN~j;h?=RYhUolSEG?wJFA+0w)-;Cf)Zm#FVdARWrYdX_R-w0p?lg(^-&@nOQXS2+hLn~X9K}a zfRjbRBne8pAwP#Pgjx>PFk;n7H;sLyv?S!;2H9^v?ipVPq@%ePa{IiSSguy^bE6U^>t+ zE7Irh4?g8Sa)jy(_`&i2Qc@7U6Jxknm^!O{XPdl6IuO4lVIYcE7kLNX+#oVq^8{=| zUCHQO$VtqdDZ+T|SEx^-v1xyTZG6?idr4E@3ayT)Qv* z$E+at81d%aCupTb{t_l^DM>xUJU71 zGNqGam6_r-Qr9w>!L?zQ+~(*a*%^pj*lC%TcGD>IH^nX5+9NOjRONpC4gKh8{;LOP zo%7!Mc4PgbsRuN&HdSN#3^McoG6Nd)=+6N!9+7hRP61c@4iEfJHUcU2tMz+kKbCwe zvXE(PrD$INxNJgkFJTg0_wQzqRc2L+7h3&dd=V2>(3y|`%{9@Gai@VY^;Rj9Uryf< z?tfIe(P@p5)3XR39q2KP=A!z~&L#I>>yNWwkzL=CaylzIs+sFyt*I&jKA3nRWj#q7 zxEl9cnxzcW3_ZK)B;K{C)PFohf3elrUvt|_>^YCYWJS?qIfIQensD?sTLy0xwaW?H z`!h3=GSa9?3S;Q+5j}7(nE8_(@`;9T^0)R?l1hkPl@Z;N&0+MDsL^WcXlp8t#Bg71 zzVne%HiGgEXs5ZWP|nBwsxv6!ScB@_xk`r5%|5s6iE4ABS@RU@a0_nS79S>;(uM-Q z@Va`9-6HuNH855_+~D?E8t1;5J4mh^D3MxWcxU;=|hdT#JR5 z9trDzezVgT?beioi70G@vtzcD5-E#{v%=?foYJ6dN2tsWP9iAC6sR%X5eX$516)X8 z)yC8&Br*`s$OG?~9J|B-f;A7+yZovF+!z3TC}zqZ^u4b9k|K~& zmma7N)+XXnDJ_wq7^HD&cyIhM`Eup+39tE36mBwjD;9ux zOgK9Jh_v~@ch*+`u=Y6@*pqM~cM1&0EWejkCS3LV1AdO3;JVg zJj8j#?u6?M#FTBN*5%ilW3-Wb)NVW(cl4A*XCki~V~m<*<64T-jWwX07%Qq2a}P>j zT9cdpX?(dY#v!@76y|?FkLd)Oo=L|U7Qr%HrZ5LoPKH85c_Rs@f)*eXlZ)s8G1h=_ z7F7sRPCU9$%9bu2NBfb0DyfR}lxqA3EOhOLg>zz4c^1Sk4+Ou<7ZFhYa};$V%ymor zPckY27bs-o(o!_yNR!catOdC#gSoA~G3PsWyy*t}S@7UjZ!=_D6d2l@z=y7%8}SCeHr`kX+5yYr{-{4X&7T@d8rU8?FZRD$RFwlu;@(6;+a|6L4IL=30E&2Baf8etj1uis>&0gViRD6&B6 zk^rK7=#f??An5){ivJ_9cOaS_hsN;nrwdni?Ji0rCIh#9^-iaot{fQttP%Lh)zFu{ zo>R*Oatl$u(2$fxTPBgSL^lqNbYF*wnF%o8ilfwWEZmWI%S<++r0_b@geuu{C|>P?0Q0sHf_g&h7?9VIa&@wahJ-W zJWpy&@)Yiy3-+k}XR&l7QkdLhVoK~6%)U3~p=dZ*FzLY~&N|2M1?7yq|Ez?(Zbtwv z2y~z7W;eIrj!^shS)gw#o9Kf*Tq`RyTx9zC*Y2U-^}=Bdv}t(;K>^2k%af<3f{|>T z%~haLJZ!=1ntzTm{j0TB<2{qpvRcws~M@g`8 zj6AbOFW4DjZ|88x$)#M=MlV3g`{Xc0FI3^_0&L*BxOlC0hWI9z@h6u%z1D}YYBJh* zEA`yF{<9UycZF@;i{SR*Uf!8a+Vr-+XV^Hy>N}QuRyP!D9RJL5Oznw!(O-J+>Dt~4 z`~+t4SRDeYm>%^zqR7E`4Vaj`jI(jzp$?;{vBss|?DQvks)f9}$(3zpionHHsVkG> z2YOo-Nvp|;s#a;IGpu?=rFZ}?`Hc1aSaLb~PD62wumbv7D)kB7T$;Y5Iz;U0DM5QL z+L^m7)X@Ea7I&62Mq9r@?94oQf9{nzhaNQ{Em4+ftYOErop^^uXay&9a5Gx0@e^;YeOm-#6pwm03NDVz{j^1kJHH~Ky; zC+S}geXHPGkDI$f2~hr-;Pg9Tn2`&VJ=uaor%X}*Z)!*&X1!ij52gFMQmxS1c0TRs zAY)s4FF>iDygqI|w{A1=Z+OnEw7xsIkmN4FdDDTvqQJUF>2j?y@<&fOS#t=oZ)|1+&ufVL_0tV-+$8bxvWYgglVDO|2&u(IICp`zHKv!~j3A_n}-ncGUV_F67% z)M5~8joAyP{M-LPjZ${x$y)b47a~A(xeLB;1M%j;Vam%5vh$v6*aKtbXl@c0)w=g; zdxDkb9ym1u*ygW10&<)_6Mg`$58H}fW?Il^Jp?t5ak$*@lG~bv*|3s1HK^&GiYg4; zPq!9xLuyCZ5xg#{2ABSzwy5j=#&lq&PkKjseQ_g*m}n6*c(S5bYM4P6@UPR2;dUG_bhO8Oqo+?%)>YCj+^x})@GQhr!Bq+B4~ET z8U`m0l*!I+KjajKML{(!RMRDLz-*^Jk!6QS{@V`M=g_tzmUUpq8%w)O+9vnJ#;-hR zi{gwfqF06OZ$^}qmh?&%XH-M=m&a`;G~*OwvFWjQ%LYS4y06Df`U9Xy+fVcj_$A2t zwbHk@C;aiu?V(lxbgLO=uxJ+!OX{I}kpcPGSRHOnO z4c6+xJ)5P}t&v$ayyPM2PjJ~j-R%F{IF(A_(Gt#>ik6{+QZ_27aIKdZy@;~gy!xzI zR#qxXN&U9%-%I-p^#}i-vQf?W(fxJGUbGI6U%h8H6dzKx6;agNCE^W+B@4sW|3OUTdXM?Jye<3_lIK9NmO<@reRNHvC3&CV>r*1 zS-A|)i^|+gxWsd|a_4Nt3N(9km+blRNbG=Z#Gm?ST=uA+04;vtBjHES^|1wt`-TNj zcN5ANGh8n`-%Wg;9pKdpSEaj0@4%-4G<6s|Qeh(I_r2b<8hdr0>?zc#zv`!%2$f|{ zj#p`{NnGZZF_Lz)K7Qw8Y@M0~zIyhGKJ`F*slM-+g~$^=0KR7oL0>yv1hqahTichr z$DI9HJDAfXs?X0)_ViPFnhR}Ag&VC}IxQ)=YE8wx+0~ED4}Z^P#Ttt3B#-u@c3aly zYZe6(mYt&ao$1^$L{4aOu$pU$A761UEA6SeRN&7oR@A}X?G1Ik)Z&1)6JO6awa36M zKrOI6qr|XeyXmvW=(7!0vC0AqUCir%-u$Q!-4eMX$5{o{`s zPA1);TGBRNJ(I_Q3M@cvl~Y1=3A{61wm{ilMZcAvNs-x+*aIH2VnwIAcy0@iFctOO^J$+ddI%uL>T zg2CKTKX(9X3&5*qf;-^O*aFdoMMyfnhUVd zM48Y~XYEr8HaB54N^OR~St_@oq=gc3RO}P#(L{>TdUSz42$=L1cpFkOU!+dikACY4 z^4Gn`v}h>Hs6n$0Xd*^@a?Dj%F=PkTl@IBHna3&`)D!)haC|as{{HZ~uNHFPqZjgR z&-J0j958#d%vJ>?gCR!{9O2wf_)n(4D(jBo^>r9rt6}A}6m(^}uL2lwJ!k13@a+3D zAAq?%`b$6p0-i%)M?#69(wzk`i^|U~q+MO5w?%y0P8gae2BxMI;z^Kh5p0EXCfw4X zr)$V_H~EeRXD9joD;GiUN;BiDCKp6c0QiiQjjwVFGz9(w|LjRHQ+)%b{pd9{+JQ#A zk@XloQ3OpXy!Acz;}xr|M{_S*cRO9%2%X9Dx__Ir$Nr(dbVz4#I$?f`S#g6SSY0xp z4RaQuw7<2h`4>1*go8C!ZwS0-Pyo7V@B;C9&Ups`?Fl8KKsVFkc17q@-%kWjfes}0 z!C77RC3Ml(%j{DK?cuNLhX9j%kd|81NaH-~w=UXo1pj_WheU z`?W}~(X{t&kX^RqAzPDopO-_N&yM#`fw?nZYbr8p*tyoZ+RJ4n2|vAB?YC4@bfSLh zD?zu8emPxhvVNWKH4H$1?qu?&~|w#9!ad%4+M$HNdet?SKh-(A;!~`V+`y z{D5>w{>z7(Vl#tYtTlw_Cbg1Wpa!CZT0~B3(w3|nHwr&LYI)qd^CL&6EKF7N2yfTQ zae|C^8n(7^5#kq_~4_>j@frA5~WVdhyf!-9kDe5spZ}T1B`a(R-ee2p3H{IiNG2Wgf(dm)ICIR~4ZZN8o=7?$55w znvVW2I}^@mF%8gmLTy7%2=~N;$j$=gemF(y&&-sG+<6$2Jdm#=+tnq^<=}w&>c#%4 zmU90tJ&k2DQ2GKhZKN$)GvRuuiSi3W$YbA&-JR`d#R7oWdlBN-lv_kVXI+;PbW2-S zJoMZGM;ae*O~IAXCBI_f^|)rkzG3W@l}+R0+($~8 zM-EC4Gwh1LmGj97p)Vr28)3bMd9=sbA(z+pZot^nXAoOawo#Mh@~$L{ab1^*D=jae zp@`g_f0x$U!>Xz~K)*QDc#%~_qw8;@wme^C=Cs6uPsq;}0UqscjaS!C?1hM9JHeZK zwk^BVHBnu)R~@*XZ&={f8)fS>GXX3(Y|qq_x44d4vNNQ&;UUv%`;nk6rtFq7(rhr` zmQr){ihMx-?*!O(Nu`hg9I?Ou0tg|U@FU+;Nsbz4_e>Of0FeCETCZ=n_|!5Qq+?>@ z3-!{%GV?#vz95p)>-c<2w@=hn^B;egfojuSc0~=>%Yr^GKefjEEYva85@lC3C0B0bc7zc_A5S&4g-sz){Pw?Q~B2IE}t^wKBgFw8f z5PRT>Am7v{(0ZBO_Uq&y{jL4Qgc~G{cgrGr{C>9AA_Z!Cr)c1H0sztX=9QBvxzeZ` z?*rArG%-jvx#TS0!jkw4HUyX(ccCZ=My7N3emjG{`$Dy|nMZvpkhObEPo6Ke8zvLZNpNUUXP83t+F$?OyX%-}R znpPw#QIKy;O4s2YOOh2qB?+;1GNfl;idz)=+4FK@`g#g5{+_zT_I`aYY;Bh`#PdOG7E4Oa;XxA$l9Oo@*ElF7Bzz!$VvlGJm^ zw^oD}pKzdGY*0yxNGtTM2Q$eaFfQsU%`W<~WGuvNz^Fd7;dhu(r=YKCT|DMP4(p9? zO<-Tfm1Z`Zxi*{UxezbZd@fB@yJ%;L`@JepektrR-xwVF<}Dix?e~QI-ZT%CnvIee z**dXeK%Zt9r}sz#t9WL=%Rg?848)Fj z_+m(YW&SXwS?KZPNppN$L9izcW886|K!TB8LQe?!)RfVUcS`J+?sCKXO&`as9-mX+ z4YseJM~J=OPX;EL4Wv9_5rRz@C%}jk2{6F&*nEB^GhsedcK!YDh)!0!EL{z)8woie zRnfy{I*jlkixZQHsfcJ0fziY?ITz#p7>^sPE2TIbSYQncyC#l7Po*p{dw8kzk7t0G zuY@ltQ6#WuC&HFhY%%O5UrlNQQM5Tt=({P9@DTU+J7 z)Pf-Eq-Q+~93ogfu)NdN1eB`J=iD=gM7cqtPkDG%+Cv)+%gHc`_3w!_{4=r2>Na<2 zlIp1**_p<4vXY)m$3Iq_^IH#8ijWQlw`D4VCN5wJ7%qGWe48tYV!F3Lf_#r$52M}xeD2s@UPwy`Sm?PI`rl-f}j7nycubrWXR|cqwWlXCA*As3R?Qm{qFua-Df(>;c#-AKQHak zzqn29^ekE4(sbD5TjlI+P#`%Fs4{#yk(NFh3#|1*9%=K&l{*$7uyYdr$u-JKRvQll zC}Z=VOsYt0F=K19F*ZeQ=KBx4L2XM(K)g^3K3pfcNi2jjp@}IK@;j(M4(&yvilepI z>Qc1vk^9&j4#tYl7pO~)xj>p4?>Le3&+@AZM_1KYIZ={G8?}X7M*R6UBnTSFVzdXF zO-zmS=|R*Ub(Vk?7Mb|ss0XQ~|D@IouA8*cz)Nv?`RQ@KbHR#g>SHWVGoo z=Q~8ccnyhtISePeQ_7kj7WL0vpT>g-5$E#5`+(@c9sNcny<|2RNpL3Fk*~hSZ2j$= zJpI~A^ik36q9Y=otQEebD{(f!8e$gP9 zu2eaz*<`LvYa_n(`y`3ZK3Z{>bP8$6;YKkw@Z2rLT>KS$N03}dfe4VAa3AJE;#mpbBVXM z@9g*m)N0ox0J+x7Dt7hkqumJw)!_MH;3`Tb>U((%Fga+R6mvaaJ;tC8IkV2=HviXM zJ`$1M!Q4+33&*uf{I~RE5R=RzQcQkngN~mhm<(FGjzRqCAa=)H7CayK^dhf(-#AM? zAfC!(;oeFzUzVYlZR07ko$gIJHbrMvyG)K7)RUbZ=;d_Z6CwH)oAnk0)TN#6=pjCW zY{7W)qJ0-EA=bN>aKqi;bU=CM@UQc`eEP9LMGS2(tG-~fyY7pq)Rp~rcT-vV6$YeW zdIbvv3Isv7nj0jNi^ppgq`*+^8dQJMg7j5p*WfMZC$prFqcP%kr^jEo6S=5om0O4K>v+Pdj6vsbvqk&&f zgc2@am~J`iQFEqclO+5gOFy&oP|%rbGta?@GP^U%-WV*#%K`%}4wj1CZfFHuxT{#o zbT~A{6h=-&iQb_{UAf6CEUu4u)*qjRIPLfWrI&P>URz-FEYm_Q#9*x~)k;^yBIbY# zmt7?Hm;h=dZ<&@MBHS{rl3^kDFB$qckdmP*;tDQAR#7z6T4bTZr)K-bqXqlBS#`l3%YuV?Ggob%5Y(AhSSG2yyCKZGy$=HYWOMkBj(eFOTKG z*@ndfTu>T5m~ET-6hZ3Vl@6G(q2g~-M6Y>^`+NarS4XJZbQ0KK11@+)W? zh~vNMR6dMDM@IikIgnkIII!NbQAm)IbC8lF|4F)D*%I)bs?$?9?vFVnW{{E&uo7Xu zd;earibQ-dIbG!qLP$6lo7mnxtXe_Wu#qlM}& z{uV0S#EP%R6lZ(xwGxE${^at^VU_Ocz=L;;2uaEC@<5SyI`6J$CAo` zJ?g?miG>JEg$Pq~2l5J4(i-Tea1*Z22|C4G3CTk?%mb1gc2L8T1W&SN*}}`e`je`NSSk^_z_{!Pabrh@Z(fVzev=v z^F}GGh=}t^Cugd8!Y4&TQQRWKCRTBZ1k%~fm^P6~%Af4Q+- z);pUUn=4<9I`qcfe>#mgQT9u_dFzx`Y<36yzfo5-VQ$~9C@dxOkyL63g=4YX%75cv za4VJl!wpLJuSnIqc<(V?P-NE8{w+x9PRBu2tdDBTdb@#xo6hl2cS4wfG z%zJe?L#U4=?)bPzO4?`?uwKtM!Fu~-vzvje@!e08Wb4_8}sBb4`WZBA@lJ~Wf@&FQp?Kc z&eleyvP~;NBiHAFGT=;fRr%b_8as{XJ_`?Y1ksOUTn-kQmdfJGo;5U_r}EVoIKyqb zDRbeOn$(P8m6Sjfn)8x9gs$IpA<^eqowuW}8Ai|tbx|>OK70hxo9sulIq*rQ=7bf8 zqSwd8F)&VZ`5!Z8$e*y2nVhJfe$3Quxz?tp>8N3oX}tudtIiHA{3jZ%aUQo^XJ$fe z48Qgs4nh&C9d=h14KuOtC-uZe8t%JSC>B+k~ z*)8OkGp=ulLa~!pjx%2bchZcANtEG)CP9=yk$Rph%fy{XOPG`!Jk}NM?-f~R&eaMZugoS5)1_gziESb{iXT*~_&OhRI znNiJnVCslQU(U6?479;3?6A|1J6Cgo5xlh{ONSm#^%l4&!81PlwuNRs-s+)o*G&CG z`60sTN$i3-m)npgR}fha$8&4)92bxx`yzj{9#Etb`zZ&J(sDC0j-{X=eU#+Wb&n7r zP3DD%R}_KgqXOb-!4U|TJV>3Z&q;jY**yZmwkAJMrSokVp|c_v`7LgUpLjg~Ms~-k zMPnZ*@j~OseoM5(-H!dPo%64JdWYJdv}9MgiReCpiDe@h!xm*yB~j#-;$y|19udD( zn;U~jcho${M{A?<`>nyHztBVSaJIbPOL3;HDd+yo&6GNw8!fE?v1kQxM#_id?G|uPj5~&3zCCO)b6_=(^2kDa3nmAIwCrTWQB; zH(l=|bO}T6Uq(H?1G;tD+tO9LzHD6XvPg;W^7zAcp}k*xI_8wx1v*T6hG5^ip-D|Q z$r7yXGPw+SqR1XC3ZHtyJo7%t-(B8Is8d3y#$#ir7Xx#eDR?L4*R``wN*%uhHJN;d zd!46#yAEgb;u*@&&Tu=-)UiJeZ;yrXHdfA^>hx_FrWcK`d<||kTS~e)82ffS+RywB z-vQ!&dW~6fbMi7o_iJAghuCBK5^!Uc379(!cJV`>EFpwZ&tS|A5LTFN7=YbA9}=}< z6e$(pyxU-+&vBo6q+SgQQVYf-->`Q~Q9V-Q}whDc=F$7(_LO;VA0LX6mfu zAKYQnC>?8Kh7^B5i-VeoW8HG51#f049jp3PvQ%p*Ek(FK(}6lHErExg3-?u?o)W^g zEIVL;)SNAY-cUM_XN0kY8g~kyD%Au+NW)HTj@&>|jBx;il zXIe_tKDd{7_n@00`2$+npA7UAV)oy5uJWt<^#B>tT+A{5UXWcBia_Gk#sYM}lgt+M zhl==?=tpDRdeuOe%FV!Qc6Y4s-!hj~5CiI76~D4IXhv2BOZ4J)mUa!!^?@8SWvwX1 z6oSeGjFlyeGq?#OavE>_J{OGLOSq%vny@P=TO8Q@fp=0v#S=F`>YQk6*Jr zORMyxYRL!aG1w$zD$P5H(~+pELG@9Tpq~y%+Y_E6UCa+qhblC%{dph1RuBi(I6|KN zb&1Pm!#}_7_b3(hNAvpEmHnD95<#p$my`MYnmiOUYLijh9OHiu3F22mTEne2%Cyn3 zeWs1xDH3tLQ8Gq=QIk3JZMj6D7?fXuyRAU|5h#sX)D#6fmkhk2VrAH0k{6IpRNCUrM~SM5j{sx zGcSz4c+@;T;5Ur{XuDLs3-c}7?v^WEbPE{X<$7hk>SwoMW#T?)y5jK=3!z_<^RaB~ zdWq%>cEBBqK%FrlcJy!$fHS2(qFcvfE``7AG~pKR-fBvOc8_WhI%)oZf?B*c>53rm)6mFHXU$V4LerdVk2$K+^|8^lVl>@4B z<8Z7Yk!@*Sh}tArKB{aWd1V1nqiG2sKq-;Lr+$JIK;nu;7QNnYxu0k=Y@$JNrQ89i zlKH*@HI&SatWsz}e94JZY=iZC3lf&1F}@awL^e^dLh&?Bf41QX{v(&H{Y1DGR!3gK zfAGJLRW6Y*91DgVC!Y}t=GF~v7!AAAt8#^Cet)5(J5-4{E#&?Ta1Vo0p+i9sG$lm^ z7gI%jwbebFg~~JKpdrV9|EgIdT|(Gbt3-vnYW@Q!+KRz#Z6!GVS|TLRPU-ZqO1ATc zcpMin7l~k{vV=9*U5IFMgtw-!xV5Ip>X}Z>!Rkg0Ppa-h+YDbl^<$$vEY`hvmGMn| z`$g($rszn`DE`oro=aQvABY;t5bA3xn_y2y^;PJEu9IT&x{5<@dgqgpf)S(dTFc6# zz3|-ydqMqZEEm?eUZa{lo6iDX_Zf!LH@YE)s=`Vk|HIm=A#}+CQxm z6Lb7?E9w9Lq04CO|2Mkqe4`6B!U4Lo(}K=eZm#^9WJ5(}UapMi=nK~w03Z95QaMEV zrtNnOIKz`qQD^=MLu{Rej+j1L=MxZU>bp6Sipc zx>K_yvV%EBPz&({>k8r6`=R6DZ*Hkw_~w?yBXS*_X`BH;c$a`y6>{26L#yB4@Di0q z-oyl;tvn$!ubExPHdC_KcM>Y+hNa&SN~ScCwfNrUu(^2O+hbJo!EGs#7_BJ|a)tkX zxvj|Uw!2+7bA@kXOUUB#IJd=}yn>v`s22lf1FfYWpgs;wEGhNnx=%0E;n!!i#vBsR z3@s_Y&bfQ>*+We2#7f)>ecDZNO#spYdp3%^Ei9nVnVXv=W22~$)j*d^Bp~P)P>-yj zpqNMiNylI)?-Q-7eXY19_+ybe_*%;rL~z4D_b%%UK)MOeUcLeY3jdF?K7>R!Dpmxc8y&0m48Scyq*RC~qo%nF;mEQ54zbJ^N zo+Oy@wK*KX6U*K~Oe{-5P1q6$dB%g#TkwD7{wB(WlM)c{^OGVZ%FQp8g7~4kZ!hJ6 zV=M{DUmfYQDMU3o8TA0)wg5| za|mR7OZF@|UOxt5?zE3UmOV_1Fo#M>3Md{J2ZTN4I|vensA_NC{6PV9Nbv62#(-a* z+5v(_X}gQx-|%@*_@q!!cvdAh)c+!L4ndFlZvR!xa3}zJLIpAnCk2uT!XZrvGM!&4 z+dCGj!|5Bbzgr7Jf!DLey#Mj7*>kx6k7i#wMWOVTT!FXDK>Y#9`qpfSEPId^VV(z^ z8wE(RUE*HhBIKL@It@8#dIt#!sqb~;vmoydvehksq7qTSJPO4x_6nS6Q2;@pG_E~D zz?`H>yTpzkeXDU`Z(}cDiXDWFh7!~oY?aoh12V&P>);o@NCv^95E7`rfd$*i>ew_M2QB~hy45%8+cRh2? zwH$H9P$aiu#zVRHc7CfXog8P}6I{@Wm!Gl!z~&5DmY|538?9`(9&jb{b9GS2v+9+; zaAcc0K;7Fs2;md@;M&Ub`RM2y%}1I?N!+~G$F2xv7#!Kt%5UPE-dZFCIclrW+=ag8 z7C%8}&%Ts(WXhPyOsf&u?-6%*p&%D()eP)@+I3RTjSy<(7uZ@X3Y9${dnZgB1_15I z8=06(;j)%ltk}DJxUD*z7lQSk%`p4q#VR7t&@Ae zfRwCSb}p`LI$g`ws%Sx3DzKNaFL<)+s$ZqfhlaWtE1p=zVRCUu9J=NYL;wq@{E(!b zl*{L6({+QAwA$BJPGvF>AzjJVI1?vS-0WVU46EwVuIfQWA>rgCH!;X3JlgWLJKirKEa!7widPvhvy$%~yGS0+wxWB6zp>EQ90TO@EK z5tIm%xy)nrb4`)XK>E?-j!!T|U=lz<=FYcin0}T5Y-gCkJ3AUO#Yt4Yg&B z4;=LS0qzhs21E%%yWb^Q4qmUI6_BzwLT@Vlte{flq6Oe87L^(Brk& z(i_N*M_pWzdLxG{9HgIf&PQrvVksaAjD0B>>jN9dZf6H$|6~afiXfaaKe@}cvq$Xx zNNr*zIB}j(;Kyu3$4g~MNG9=PWiTf!DtxbMT-W3z5AwFpEyhhfko@DR+D z*C4!_JkS$bQ62kJMND`$YtWIxP|;kc1WH&#(*7rqkdfxkvS+u^*IBKoe)9~;&>-=H zS10BJy~D(y=u14*C;jn!m7fRLW~V9iG>L=MX*br^F*{wG-RlQJf?8X;^r}NEd=z8= ze{u)dc+-revV;xOyrxvkwd`hcc_Rr4(JQM&6JSh%oKn(-`8J#N{p`gaw&<*@u|e3! z2ybOA20Zxq7-y!zCUyumn}aV!Jb8hYoj22aG%g2IF>@)`>f~6xg$#PT@Yluk27>0F ziw_sa2eu^31D8I;aBNG$omq@6ICVl8z1v0Uswk}w-bhZz+R(xcw0wsXI0t(^z-PLQ zS~`AW4RV1wgVpEaF4Tk-+jiGc}oO3kimXL!@hSw1zk`dceoZK zojC+43LS%hpLqgapztA)D3cuFE1VB*2xAxpk$##6O0oYl8EI_f2N3%f{MfbT4M+w# zdF8_`9~|OTQaKz&Lr~2bX3ZOke)xk(XwXEyIz$ler2;5V;PUSiGYUAjIq>_LJX44V z7_k1@-k}fD>GhDTqQ$yeIO$UM+dt+N3K9H^QRIdx?zmBRI@z`VT? zE=ZxRQc}7yHbm4kR6)WSXMv4WR1~0#EIPUS2gn4DZyLAVi-9*An0^-L)ytoD zp$74d_t$VlKi*yUyCE`dMkm1g{Zr{jrIkN(|_*7y27RD9JUJfu!7v0FJ5ci z?W|o#8X$aF;9$dxKG{rJARBWHjh=wtc#gK!?Ss!v(?)RDRg z1f#^6-n#YGdf?Ke`*LYWnU%W>8eCFGY>lGG2U@baQgZm_z(V&){7Sm^#bwpt>8c|- z#PD?{86Bf5?wdm7o7X+Y+vcvDFC8dN$~+%`M-Q$WUv zY~=tk6jmMgBPSH<$sV#?%ARAIM(RLzzxsF{D~L~}Q5&jHW1YZxI)>TP+($`!_Q6Wa zDttlTcI!T^%E)xv)^i>Dn5a$?0Udx$qKAdjD_$rJ1JFN^YoUP z;h%pvccD`r%@_58QTcsoh%N9%{uV4474S{-a2%wnq9V@U-)~GEFdzn&ms$+yN#Cvo zm%AB$0f47%SwrU!8td?x#4ImyK7EQdo{w-#H)ZVRjdDXE+;~6YV373^&2|mr54Hg9 zKXmdMr)@Xq?j5z1LWp=2j0cgzw(#Vc+2IX!|9Dig$-Pdrx?k=fY|*kXKYg?Or;+8pgs&6M(HWB06L_roQXn_(bigKGcI65&$E`|h91$vD!qroynrHMQM68SbC zWd|KuAd|BHlSyTAC)QZ(xO-2b>yMN!5mN4mL>Q&o;0SNe@&<&!qRyQ_qrpY}o|~O! zmYB%HJ|>gP%=-5r846dKCw$i*rg1V2kY&t3i~s8NHg6ec4rNC*2LtJ%K_5i85{}!{ zah7^56C`gfyZ2Bl(Rqt~#_%!H*f6x9tP}r^#EP71|7URC+1S^z$!W!5wzLm)kqpr? zm@b2gYPY|KXG+;dAfIyF{NM1Y@~X4Q;njcQQ=Eps`BZ=f#y{rbWR7{bu_Bu^1EmgS za~z9izCWzBzwoIbZ2I6OMGvWp=T7bw@kXuT+7K4JtjK}w>-b4nJhy>4?B}+OU58wfLYDA9T{iv8 z_nlk-2;;GgQ15QcL`_*1M#jeYzfQ&_Ll&mCQQ%!=2*Y(1@t3MEjh`)UtF5vZihr2g zEnvaRNPiXiv28MU(TlhE@AI&s)*pasgJ~JR_k|?Cm2#6zvbu0UN@(k$4dHFZVJSqMU5F!OO@Rp3-k?ydn!){eN7b_fFa>9wM zDw=a11IXJbzYJsuxj96ylbGs>==XM;D1OA_J_!z31knHpH$_1dSbwb@yY>C#gwYt8 z#lC{%d(zWE_iJEYfZ} zY%;@7i5${U#FIPJ_F*9gI(a>q2$yWPr3J)T#Chz0L=E}M#Nk=Qhw|C}La4$j2L9Lc zu-N}R57W^d<`(7;LaeSZsW6Zq#Jf+OP@4R69(SmQKYZlNALB5S$FFgi@?(f!Cq9M# zU$ePirU3K?nv@QrOoyv0_=7$_{JJKie}{8x650dgrEehR_E1PCS*exy{7mzlmn`@G zeH{Mc{CgY**-sap4K>dH`#3!P=Q!-5XeI3<2y5YWSPZXSMp4LDSzFUE(H<(HK=jmR zH@NU){~?^e9f86X$* ziD*sr`!kYmogYw=*L!NdBhsD|v>6h)6{p}4!p$qGdGo`tcE)*%*pW0FKKLD`kquN^ zF>n(Y*q>x%ZPx}>PRCyk9*ojoN{@%+Gml@yw7WzRt34~cAlct1^^p8<#%a`%K>RLl zJ`_+B2aTu901O4*=;z}PmA}coBx>i%gk6Z5a$}C4-q@@~-gJG1qSf(a(EgAmCBy1C zsvX>9i97sx6E`Fx97f10V=3!m=G>}e*m^k5O8*k{jk}RGOU;ov98~VzxA4|pnZ9BiFbIEWKqs4I0JJ+Uvuh4dIJEn06$swJ>MgI$H!iL+ z0uc0@7iYeZwmAbI36S!~#I>0DXxDGRJNXQW1?diDxbK3D4Ndzrw(`0xjVQQEhYXhIEsF zPq~Fzi>831xd^*yIx13T=C)A*SZQ{(WA76|9J#S|X^Z`);JMz@J{>)DZ<|{c{7oSP zalZJtOmp9-g>RL2jbiP51O7-P7GN9p`%=W@lH;-&Fipps;Sj`1gg8CWlQ*!3PLFpR zau7cK028U=Z6VO^=cn;!9*}Dbs&d^|#AH^8xoYC`565rlHJwciq1fPd2Kd(Rd!tIJ z5x4p|TU+ScZ)SkMhk$L$M>hmwR^UED&3bq(C2N@vbRHCpO|W3~-;29r(}B{w^-eUO z4Ed(8d`fhnusb^4?TjRHSl}u@lsDen)!NJnq#;NTjdY+RzCT= zdJ{)NH2ev>3Xh$4u}EO~KS>N3WBV^&m=QOA7wO^_Zw|Ad@sY4EOd9p*0VmuiuHuN2 z=sU)0%}Tuq!f+_N3xV65)R)?851w=#U*6FB?#GCv4mfXcW0c$TR3kc4& z4_NP(FRwJzywoKI^9(8d5?dk?TjJ^z_0f%9!hpTF>5OkPSbD!jo+ugfA0p4kIQLuR zHNtN;CRpbV5reJF;za<49213j77A8gS3jo|j`dqH1+2*hSP2FZQw(JXsad%)TFA3Z=pW5UK3&M=p6P0Z@|Y&cKqv7QwexcQW^cYMrsbXOuffyZS$pUR)66yz+dr~tzlHi_(?4@ zR#pa1bnw7<#pc7!f_HF52!=0foGNGindZFvKKm8P@ucVClOG*n4Kaew#- zR(s-qQF|2=k$nP=@%aUJ0mAK zM7_Pd{7{JOFwtJlj}VwXUKWU!7P`080@E4j*f&14(046qoHo)dR^Z<+T4h$EZYrYB z;V`eXgj-Z^Y(?v@NDt~)+jB-Ta~Q`s*`HuTI($Vly8XovshJmH+gi=43}q+CZVMKb z#$;tlxePsVZ@a#}cTC6)p5bizn;W&6|I3Z4{rvt0QVf~#Ut%| zL~SfLc6j|J6icb0&L3w}&WUQcAS|cX7FPb&t5*`tvd&u8&wwq`pn6k(6nANw;@glf z)-+4-^HJYH06-+Za45lJbtI2T0dEpzRv(lmq?v|iRhb?bg-IlDASJzxMr$*ZfK&n2 zXp$k;<0bXLsg=@Q*XSI!HkaTga4{_Pg0Am#rgSpie@|%+F3Nu~-huE6d#(gr(#c

a6b=pi9)>G(F8NANK&T?Dk>Tx!K!MPty{+R4$U zGVf`Ybc+b*QKROkOHFNYJx{juz!gYv5NWlT;#|THIUc6^w&}S7IjafIKP>-XLqei6b~()3)MwsI=Faktl~DjUEBy!Ay#{ix&_`nEuYk(3*f z#p7U~`dVr7WD-Gk(*>7&Y)<&P@-|qB1b*CW=qcsZRed@70zvi0kp(p_I_4Mg<0l}peh;;^Q zJKnjpx?H}FIe%$%sy4&fk+S+j9^LJt?;BmrpBG23Il5`kPVAE+_w*P?tk%{+y-PQe zfy(G<>C7ferEzxv78p2zvj@{k~Kn2KZtlNOMO7^k?4}9guYw z8&J;hjd9hkj&?Y@w^znD#O~*%A}b>Eq%I2xwLShVW_T zlusb+YL(M(knCz%JjAopb&POaE!F}sFod$IAiLXi=x<;^E0tX_ zTkc?ms+UqfP@TpC(nVxa_}Eb7S!-5a>NU%!bGQs!pIUVga9rMS=8uq|XbK}NLSI#-;3q$EI0kjp zf2rDp^T1`+)bpvJI%$3nviScK{fLJQn7>(Eh3%pLQT0_SEv){9$a)l<_fRt3dgkBUxXsY9-1W(}cBEuo5hQjI+@l%PmIfR%J9 z|CH17V6?7{*kCG}^w!&&-Bhdo0mfyc>``R1k0UKC(qDC!4L$k$cNW;V1`OV56q9YoPo^7eWo99Y{0i3|^O;xc01Xg8C>=WW z$MFr04H@0Nk+1{k47ou_BSuEX^O(QJQc}^Y^S#dsP;PaP(M0Si`k zufI5AGM_!3->V_ggV+(*x*D&xR^5&{?MTeEKGxlX>j7IinhBQf2iPyaMgsQ#Iua}u z039*7z3-AUx&Q8n>9%_J*#CCKI0Cln);PXmB>zk`U?B>v>txq`{l7}6)LxFTIn%h) zO3mmLfZe+B-^pol{XA(jz53YN`=ge2a^ro-Lk2)m9t9qyG#eS-I|uRwe|888n`HMA zR8ATF7Ri`~{dd>UaJl<`4FE*{X8_Pb4+H>hoP1xtj3&N^0~BO2wGUy!4Hbk$R_DLY z$NGI`V2Z=){u=w>{F5Ih z{D&VVzWNtGj823}?#Y?ua$Ms>Xc>)B%*E7<5H{aQk!ZT@Iz9QJNr*Pq~Fx5yZtFXo;S#(o=BzcJR40Pad# zgfEz^7f4|#;Ox<}ke+%OoN{f4CFRA^@#1kV>L7U4tLi{60GkvC z(FTqAkMTlz5;$IPs>=ZZaZ^Z0_)(drp0{gZ$sc4vB^*zC?FXt*)J~?pyK7!a)sX+~ z`n>i6u|y8^#5M_Ir@z3hP!wTv?p14zCIP%@Sq5 z4b*;@mId~;Yr;}cR%d1jXjW&NPnsYUuRkrmL$re08*q)k^0-a*)-FfTjiN)W&+jj0}*BW=F6&UL~d|U#W%rj?ThQzgfk>iHS@Clru8aQznB1|&= zV)>~8H<9)Pt6^D_C~Rn$*#~H$(pvS$x*$;>K5@}a3%rj4j&he?&<3WQldF@Pv*8=zd?y1uu&Z z4#PDVTvEk<&yhO!{gET>qxpzl2nQc#lFt8K^aFQKD6yOlg=%ncX(H}0{`;3y7 z8Y4C9N6F4)QP8IHBU|?&EU*3i6(mLedyw=#3Qm}jfmK0C2k9w}B_YwlP(aSbWZ*+? zG3D4~0dGu#o99rAL=i%CYQ{HyH3{t%qIu(R+gU+TP6bPh7b+W1lBfMC`*M0QduH^^ z$_hsis?e=aPs#M0b$Pu*BTOv8`#Xv18jKVDOXiPF+q=zmfQGwRp+R0SDrDs?yz1VS z&&?4*3yO1J=QlH>5+e$gP?RYurns~V&E1`A_>YGcn`hOW5^7X=8hs1f0}9*Y>Kag} zhoIofQcOpgHVgN>11^Uq8iOFlCcj0J@=*ZfYZK9dXEcrkx<0$Hu0wz?9{ zwuEk*F|aDDl)+PTH`jvs{6|!E3K&%d_EM>sf(lf&EG@hRO78w1Hy5EC$d+8^5SYZ2<9rtB_ zrOI*{(U6a==ARK`aKeSSO05BM#pXW|Eh0OUc>gs)&rzRwDOmCoCHrLReXkI zg#T$I;LG^iNT8JPYb3}Gkp1V8KrqgLFXnQ-`L4SON_m}uY1FNj?+x8UH@bZAde3m0 z1^<2aQ}3$iIN+fwhGN|)FhU41go(<=pL@CE-Ovgw`}RnFeYWkwyU<=(#grRy83d_` zQ7H$K0DtaMve7F0V1I@~g6-pyHz&iK{P$u7B9hqGtQY$0DrG#sMuMo}-v1bpkpS+6 zQMTP-I@R9F7xxwn?vpQq>=YKzr%Mq5U6^t3E^mZHGS5!E3=CmX(5eIzk<&)5U??dm zW@njl90IdRPBGFxZnL$T*H9Ga zrzx5A&KU{u9~usx3;ffBG*YV~bJ1GAFoA=9y0E*JcGu@zZOFj!-+h~IgEkP0UDQbu zUybQ~_#WCXO1hzGEh=)T*q@k6;5(r8Q196-2{}z`fGVqp+Wwpa`t5vSO^NfiqpB=V z!Upq$K@69vFTnJLY?i|HVxx2YbKpDzP`d3ZfzsUulKtOExqIbuB z0+Qm@MbE+M^y`PF;AXM^T9Xzdc!y!;TL!N=+?PVVS>>-gQ^J@-XS2I5h(=nyhZL#; z-+2gl_q5z2f`FH`y=fHv3j%mSuYYmnEDAw7X9jBa69Tu@1qQ(y>R!kfF07v%eGj;v z%Nv`!T$~4!K!3RCt%d9W^ovMxoCZ}6HnnX|hmvX74 zj)5vT*~Wkr+*`bI4L~`b?DIy4_c7#3!~Kxdlkb9ewN8ZDt+&|sWdwBGcQ*) z=kdOuJ9V1Y&u$4-U!a=-_khymkI~T}5_mWgXIFN}3o!_G0{)FYB^F(?m$)nHK`KF% zrJG6wE!%J|n%KXn%Xi#)sXVLmmbELfT%e{8I8~puus{`I0X3au{-tvV&ad>BrpH+% zgDdpJ52dq!!|={(qC?~?0itw6`3eajJgc7YFtVx4)`eJMAHIFXx_C6`0KC&HK|sv= zLU2Qme*-u4B65HfYz{Pqkk|urDbd07O6KZME(?)i1a4=nMH-pVKVTxp93t*(>$(uh z6$<+S4S5*h**^?>34f#;<-U!>umPFP0cArcEOe@BVf!lj zioJ57Z!1qRm*CP;K@=lQnhYbB_=pO3=pXJp(#S7&9*rAtE7xYI=0K(lbmuMq=FYEz zYEo*?|GXV3pzTdjXyj3ZiT-H8u+ALWe_RcPh>B3L_?|KW<9FqCsH4P87G;!LE~Tu{ zp8&X@&@8!spZ9+*jA-w>*8NHM3)%ic_ca%&fpi~9;$P{0M&>WN4@NA%cS*5LlUi0k z{~PWfB{-sMrc-=ZLM6DAtkt`z{&tyok}WY@*-tz;tLfNOHw9XOA5L^dZ{_bT5mcCL zWQw%9Oxy9Dye6V=UUlJX^7_!xbQLuV_>B}nr?m^FOrm{j=nX7VXkn>OmY1LP@VFX z9F{n>rk?w9D!9(%6%k_KRVQ7ARlGD4+qiZtF8ml@j`6-94(})O(E2+V-@ofN2N6)W zGlt(eCV12d8%^r14a_Cq4Ch3*ba-7@Y68a@K_y=x4KT6L8Z2)jKD7M!V%; zb>#+Gk4F7-78hy?n*$*L)I`yqZ*gP3e}VfK4~{qQ)lSP>UeNy%ZbcvY6>gOP;Q;nzDJQ>}|JokN;J@m2pqLjP>8`CF z>65{~*JN!~yEbb6sLAI3ttM-~mof^IS5b|S`hS&eRm%DJkFp7t$FHI+UQ#M7sMH!^ zvMcKy30B4?Qno_2xOg5PA=aNgS&e&OPnJ^^e&{F<@B4_52r?`;YV6OCpJ5nY0dsf@ zbKOT8!9{GeEC+A;h$nW;ZfhBqOHhHE4%sc(hiaeJ(d@U|BW-IBHe-x;r6x?;UHDQt z_)Id~9PdyOJV0YvCxys5rsA3Zr>=>TLi$MmA0=7!u&gf88?ajHUHb_d_$x*oKp8;x4gRV^P-Ipeb zmyeJ>e9K|U0xspPennfUzqB4E)X^nb99b;Zuqb(RC{}zzNH7yel{4_N$c05pE-I$Q zUg-Pf?|&R;nmM@oDg)DZgP0dkw1cse$H)YJoR)*B8#APIFnb{jd2{=5Z6q9L@M&JNxX(?+VSYp?db@8Ab@iEm()hyIY6QiJJK6Fu{;OapK^ zKfY9XH}m2W{cH=YIOGKDB17-u!bUs!c5QdI?)wcJfM0e*nl*CHy~@)=>$<8B-slDw zXL6#z1{X7MVPu6PzLx<$E5AO@k$9w?j@nNtdKiW!aH9l^o6Ji%CPd33ktFetn4t86rtT@;P-hx!FQUUxFbGJVjX9MB59tEF8=#&1 zBGFMlLSPY8YJ)9@m)KKAagp6t713ME@Y(FYV}O#|AP$YM0ei<9^4YJcM}fXWDlnjl z$DIRwyB_}7R^?H?OXo0*ba@AS2pYI2BZDGXNO;XiLr1y(U3&Zaq^=|CFcVB7-UeN0 zcGpu(zc#PJ)u@i~V)qQ{AE=IDc6C@XOqQvK7b&ie;qttOi>f4EKmI~}O)`rUJ^iK@ z@P)Q)*L_vnjU%J2ua3@`HOoqSd=3BIg%US))JmqUv-FB!Tc4R}mHRVMN79V-7aIGQ zz&=Q1xoL*7WmsqMv*Q@)Dh?LfN)G;ZkPrf+`smUn*$@JW1dc_>>kTCHY2i2`G?n}y zq>*j&h5K)%JMbS4A?Hr9S-(MdaCEDj*Yut{5NozWeerAmfl2h0xbZbatSZ_pCLU_< zof|QiIyz{+dEi2l*XoXPe##}f zPuMc=1#7C*GwycNb`m3mxW*#C3Dd~LZX2jh>R~TZ;&c|V(NaO)14K3m(o$4rn=_69 zfGq0~AAPgX%xA6GY|#sPHPuGMNJ|$k*3!H#?ZJEh86G%?dG^=esqKzAyI&D*7xlio z>Ksn6EwT3Kt&+Jp1w)_SZt(m5_qet#aqf+@&A%gW;428cLtsEPB;RZ#8^Z~)*Msb5dg&GumVlOK zaAFa&jid0nKPPrj7XguGNW|4NZP+q4mUFfouD}#&KeJ_NUTAf%*X@=k zM~OPo_HAf$>OeouRvgaP`F@7i*~{Qrn z)Is*`{t2*>-Spoo*(GOQ!qhODJJ36Y${XV;bJ^8|>>-L<*T0H=L&-YpvZZf4d*NaD&bU zo~Vu(z1tA7##4+FyP zBy5t1@l`P!Yo|(X1H`^K0i>1c5X4?(>WpiL`&+V;_}7u6>~`-+#1mI3jgLYUW3eb@4w|4kZu z?^ncV1Q-w=OzIc%l!hRR6MXU2mNPcBNFavs`b?2-^;!N$aqLW8dsd&seit)Ka7OO> z-#Y8vT2~HXp4bK<^fHJ6R5OPkN%O(znDQKe;oyk5?xye{qu<#g0zeU;7)zj_u}VHW zaKoL_>DqT$pZ^gJ=4pnSd9R)RG@lGA%Tye}-pyuCuAmj@i3Qssl7?8v=La*`@qjr~Twm z$!A-&Sp~bPPzvub{**V_y+W$-c92LyrjmmZcuI%jvX&AAFitdVSec-*^)Ch@!3^tc z(btMwZ54*0Y)!S%NVHWBk#|0{GgiO|5vr>uX~gWN@8;$uHYazZwC3>#jqT0LbYN6C zzV;e0!|^E!=J)ynM1tg>!$Ye-{u&o#8~BwxSI`V#muma5nA1k-*W)QirEA6FIWczV3lvM-(I8LA5r*Rg?~ zcKHA77~=W>D|9fb|C4116N44=w+dZ@^UZURJ>NtHJh&I1eD4S!8PTl_mS-qev^GJ$ zO~I@|Kd@f@&wvp7Z-CJ0FMv?~DnD$Awm!_~-xnHLv%eM^+vzq0O5kbmq<=0nM)?0& zXzYy0)iS>zlSJMYW;eN%T1XZr1l4K^z!-6CI#T}h(Wjj2xEx|COf97+ZIW#Yl_-1- z4*%V&`&R0)GqBZ?3z70+hYQ%NtGcEVTVG$ z@cd2EhZd%_Pm*Vn!=$UMHONK240dxb^(=swJE|9siHVMxAE|F28M;9o-@?a_!z5~P zV@ypP%4N*i$j+b2#CBKmsV9-K#U4s!>_{~GCXvzTav3#WTW@+9UE*uqQfl_XT4Z$F z0@VG|)w%XrBUfnUzH~oE%PLg!GP`ZNRU(EbepU)%FYwMBrUyDN9KjQx(yRwsKNEn5 zG+Swxq$*wsL z=@}2W{&?b|8&hwk$GxF=L0HAp%8goaG>II#&d%ZqV_0TUrqvRYUAxV? zln%8ZKZ-euA>YK;V^^Lh4hY&jVx$7gL6pjXR7sR05C}i;jps4=zAVV-Vgs;f|JFL` zXn*E%IlO{>kYq4hiV{DGo-BPue7!SHX5x`dqM*Rm7&@0+eY@`~C@M0d`ROQpSgT4! zlg!?tLHA0Zv6bf*wrMlWL-%N)&2)o$;52sOJhBP4Fyb?XLS50ysrLYF)Q$SPJz9Qu zARl+;c3QPXjo!Oe5sW(lZji4vW$qO}*p!tO%)dDMESztk+Uwsf2|QgKU_=9Y`+qnO zupg!>cMb{1Ytv@FE!Km3HSPE1jlX%$oHmTMK96T=7sEehf6D)`>vMMkQpY9GWb*S4 zrxN1l>9@<3vB}zc_|%Mjqew7Aw)dq(od#I&CsP}A+Lr1*#UpFjVoXo86T`^|x6wS_ z(RpuBVC}Eo4FfW?gW)W8l9-z)3PC--B8Hj};FyX|96>UZOt4e7VH-1Wgv5nVnpoQv zCL-gWrC{vcm@Tco3O)o1jOxTRf-Vn-q|q2p*UCeOWH6 zx~3o0%B(0EKMeYojUosAv@}PQ$6zSSp%5iS@M}>2ZiP_bSvFrx#j+BjDJol<~aXO z{mA2|@|94kuamE_$Hi5boWfy#-}>9tRs7}az_m=jRw`M+^U|n$e`X)HErjr^gU3^L zdwXj%-_@qi*;x}Kl1r9QT*@4qUq<|cU{ZGw1fgmvT~^cURXpb*+oFg55ZzDjO${k(Y}&fN26 z-QLwv0-gfxbODFvo%D&D-s;nVzaMj)IJPnbj~v>@;!`%?TQMQTOLiG&-79U1AZOk2 zQ5o9Nn=UWk8#I7t;w=|=97fjKEJe5xThq z+)z-{$R1VYs3Et7>t<(3GsRGvB&3p`D!)4ylSupIq=!G)Hh6K&8@aA zY<-eS!aG#$o!2}7d4{$=FvnXC&9;k@WX=5c-Fxy99o`qPBlHIL+X#dVe*okRZJ%R~ zH$?&D?J(gj)(kDpE9bFyp>wS3$}=gmON8ikUd0%r{>#0BB?)U+Xv_P;A7N{#U&p5W zAR4$OU`(}?y4CU$T+1if-;?LuD&^mD^_6rN(p}HkQjFkGXu^RffB&rbS-OQ5^n(!* zVwI0r*oHk^%d1eUU_!G@cLxm{p(=ZM$9x}ETa;U=&^G#)>w{sEYK^D!t=-RA@D?z0 zvVF`2gE5)ZxeP)p_FHOd2FL2sz+*ZKg9@dFGN;vls4u5kM1QoN^0SEESjxXSG|=|g zSuBhAdQtPm+{Ny#V7+HRq{YN#=_6Y$JMern+ZiHO9ix17Tt2xZqjR4)=P9`Z2tmb8 zC~=QBg=J1|ZqBUsw^9@5=#AIn0dmNDp4}qMDnEaVoDHm!6@TC592PHZ#aE=IV2NBx z7!6yZad+H6qoLz>M<}!v8@UEbZBbQ4}3H)3<})J3kDko{`Msx8?0 z&XDFc1Hpay^R=cug+$n}DC&F=rL{t8`r`d%RfA!rONtH(Uc#f;0gl(4Po;brBDoG5 zq9eem_FRFbGxU~<{d@~&S3D!5E`68?A*@&EP&y}DNR8f!*wjMo(-~8ObIgGJ$rbL& z0f+1OBf#@3q<1zz)B=4oPojC3UidR#_{e*;%7}&L{J=*~S|lO>n{}!Mm%hT14JWC* zNV>o^l@+{nT>o%slBx38KF3LQzY^{2`y;PiM!<~8UdUXz_>T7616B%!9p}8!6O88w zZRSGyw;fXqxq;M<*V<)n-qX85Q1k4Ye4;upd3QnK>Y*?Bxa(25HMEEGx96_4rHZ2~$_-Oem+o{nv zgr9RmI5KO1Xc6$LI2ig?foN5Jkz&dz-wOAwYC_iPxa;v&5=Xp%+nDuKzCx+sE6FhYV#6Ri^tAX+q!*>mIj_5O$^>BHT_m` zMBI{6_7fxroL9$&P!Hsw4-GWOhKLv1QCqkx>Ogbf|8n8L^FvahyJA3^a-iL@>AM%4-BB&}AWN?ZmA zrh%^%mD~{9YJ|_neBmE7%x7xG=50v@LqY%zOeX?|fkrypL^{L3$KhpF0tG%oz|Z*V z*i4WyG?&E7YJ-8~Ep*buDh(~c1w}3`cPN(A@+p2$*rt9E>V`!Fq=SIh2Ka|v1L@$x z>Bt0e2frzz{PhUbeL5D*Jzd1+&1d`)Eyh^0a^B0*T| zWGcy8KNWS2fSqkp-n>U%6u};vY@FVaGT<^Fc}n9L)^yAm4J$C@Ohr8|6y=)2(qO3W{>k!9!OV4%F<}sSl zKtq6x^Lie0tdZS7jrdxkydT>ruQumX<6at0k3g!aP-^>~TDnHNFR?LJ>i$?H{fzRg zK125jid0L^%xLFT&m@cF=p_b|Lau#@ZAqT*Y{2No!d zr>V?CRP*V}^IQ_pi;g-U8DAyQ&+WAEv4iJ~K2IG(YofHnhDYp`jy%MuqER)d<4Tx3 zsQw5u-BVwB%)?&7es*L2xPG{XEJ=u0S_is8K=H+2X0YSZ0w+PWj`2uUPrmk0)kHqF zMONAu=3U6?JQC=jtw$(f#pLpVSIH_ISY%69gXZ8IklO`aQLEy1}=d0aNe;P;2K959)x4pSi)E*b}MthL^LVh#)&&Vj=s2x9Ou@hFbwjkSsdj?YuI5`=zM!Y7#a`jo? zk>YYlA4L>l_HtT$X7(9YXP<(ljq#7#aeh`YqCv(q!3aChx#9le2tjQ|Wt1CNgg3@h zHyxmAS-LYE)_nUT3gnZd+t5+BUPbNl|K}8_tX8oWxmTA!snUR)+q>+hcw@t&$R1aS|00) znDBNN4+p%f&X*I~ZH|dOypM;2Q z$DE(%PU6O9aFTUyc}F9}sdCvKb>WP|hF ziIW!Hrm=fnE;}kJreG5Nl`(6p=O-$a*h8OH<b2*ZI}sl!l41y)}6HWfIQ zp8oTx!@Aee*>BU%PEb2KOy;f<5DBbWA5DI^uoXm)y^38gthYs^Q^#{A1o0$u`oik& zG9U7X3)o}EhNivIP;rQSJVxUv5I~E>v@J|H;92K1Guc&g_i9h`mw7?I5o{4yM8?DB zu7-D74d?9p0>#(|Z`GDwuW@a7-hn^cP{!G}u(!U*IN$nBbM+?jRIDAAa+%Hb31i7L zR9rf5AMgGuo=;5)gQKvEl+3!j7|)Mgs4@$Yu{c+i#tWB-Y{IsY1P%1{l6~rqm3ri1 zc-*p!Yckx5rv1h^=J%L!!0~MtB5D9d0i5_w*yG5N?-H<&|3BC)RcG zS@ZMr`yYx)Ejb01_v@5q_q2RBe^4YO{QjEiHR`mgcuJKHJCjB~!GR zA5DF+Il2*$8ox(&)Y&^5C1w>)I~Doi{T7;|W0$`Sn# zlg_Fr1JA~@*3l)||BT)7OKb|#8WZtm-*w7v-}>(>fLSa--cFS&zQ_uDQr}tnZxH0i ztep^*yNZvq$qVgFnWLw$9}P~Qon_`Ai!70nuEae&Vs_nTJd=@pU-FfkmdIotjl8K| zi6u^NiqSosyqQp7k{|QaF|BXD?)L>I35$&#oLPBgu9{jl&S39C8ms2t6LJkj!-YkW zayjaohOcBtGmEZf3`2p`l|(k!P)5lbik9AY|VJtY<1 z?Z%AbWBZ^Rej{W-sAT+>{-EidCB)wy%{XF(*T3JZn|yU zP2-BhN>97~UAtdzg4ymuXD=-&oVZ#2u9UmlE12##HlAZLOlHRS-^~@Yxw#>lFrXUx z>-+FR5z@5af=B78VnfVt?W}o=VUxe?Xg9nUM}D5dP%^!Y5ab_*MwuEk3f~S=;HVVF z>}9Ho>XVKsElO~8!%ssQpmR`!l1&8L^hd=6PuI(!Nr;WaRB`qPiG+G>d0q$nb{3`3 ziz@me179akOXlheO-uHbUQ8BF20o!QI2jGP*j!UrTo%r5lyH)LlV3*So2qYEIp#^K zeS3+YkL1JU9yKJtLc;LF+I>gBBAfO=SdB4M5=Bc1>}RId0d($r(O zQ7W4xtbDu7Lz~7P3@cpDZRB;1Nqi`wu^T#}n&!`N>tmb4+%t|{v5KI(Z>wg*P=n&u zmL5FB^_fUn3Cxu}Q*=UifjzL5ZbsydgJ@p7a*#{#a|?Y;U4MM5&u0Jp!Yl20Ki~P; zx5ABUn4lozF-=}QFl(5=R0D@&^+V>Myq8>0X)`f67CR4q70VeW zsYEJ=oDxKK- z?V%P(UB~0x9{2;s`9OG2D_*<>qq8DtXW;BCz~15A#e=$-jN<+AdxvbRdlJh; z2+2iC-QW$B{@XTXRSXxj zhmlqTL?nn6-67f(QUcOy-~#lMLb8tywtAQnqr{fbWRn>K(uAW#Gt>a(98CF-g}t@IWVn@06fLoLx7upL_nxOT#ZnGBOG8#RTSVU8U(lme;*|P9BG5* zwY-|m=<2P}H{TPDNkRj~;dnv=>ESdj$!p+#SmU78_(5xZqomaO)~6#N-6p@VO>PW@ zzJ-Qfb4o4|>ac2O1dYDs-KkD3?F9#KFsn)?<%foeQiE_@&sl?j)l$ZJ@SU7k zyw;Y$uUE`xA5xZ?0x~(bsT(Hw{P$dwhpzljmTW`v8eW=HAPtm~)-+ULznCl_tbu|P zCXDOqC)6eBrYQ0rf&7SZw%DZ});fQO^E z%?0&^U}x%a6IN~ecc{m|)ZhLs7zz^vjqXyK2f zoaZbUhTM(E1(1CkM-dshs3VXcD2Tr z&ug3#wd|{bQMi*P-jUtviNFd23SZ+g{R2c50;LO=DRNPkO$r_F&q)8s9a%cjDU<+r zdyd|IK|F@`$X4VLT}f3CJU*#tj?EPUY`;=ha194ijO{+mwFZthuLCYV0EhtsZkT{e z1X|$cWa2#Z=BEMMuh$I16Sj4dK{=owgAfzqiXZlKy58}^L>tQivsq+NMWu9`sk*XA z_HD)US;R`co=roZ)PYxko$@R|w5AOF2Zz)mrVkFM$V0>+G4wjF4`+>bXMCAHS*!@X zZQ5diE)@`tHOZbU$Q~>uPC#jG$~`|KGZ0pxG1Nk6>jQ3o$b;?;o=ddI>L5`AYtKa2 z1e#}OYutylt$0T%D0qCP*UuUGzJ;VU7lxuso<2LliI;(8Fe7zcYzxrDz4X1-$xV_ls!awW5W^`W-M7+0r86;lDwB zlezs`RJi@xziIlpK(C+1ntX3)6_yc=F1*38y>90vaU$Lv?dEgc{0n2{$4}IqA{1OS z4cXIFwb>Yr*-ld4EmAYjtJCO>Mb*h+vYFmI?;{GR#zAI1o7yJ(`MPC@R#~UF%V{XXr^0Af3@YM5`IgU&Uy_PbK5X)+K*2 zgVi&SDspsfu#v#ls~mis%hT18uhf!fS1iHU)6>_;PgLD6N>)a?N8Z=tUcN#5ji!H{ zrTnKzv0AP?AMLZ=WE`d2C}|H8$%2X7V(h%0DyVf%Rk&GQF)l^1WB)}H7YuN7)s*pt4YDUAH)stX zEU26calG%|_^}+wyq1;lwU8PVt1zQiiZ+Dn03-MWzphkFd94`&rvl#5ri9?q7 zdbW7@tcZjz9iE>rB{34R+lYA%+GTTGlYT~-w z9ydKE1HQ=14(?Rfg4Qu`$O4QDEwiKpvNHbK6YBq0dxB7dpG(qoaHRniM=(dC!xW2e z--q*XMxPI9Fr=s)1GXfA=R|4Rm`!BH4xnF#^L^HM9gD zBkITJx^Bg{>i9nDIHgBw^l$dPcD0CP~ly=E4p2rAF^q4{K zpOAV6I@Q7n-+Uo)362ivj9##vvrP{#;Ah4;d|!9C&w7BPDlq+MF=`cj22@n3v)QIJWD_vHT z+<5E8P1lQi9QX^(Q$EBl)G~R7iLAM`ri{v{ZqhN^eyHQ0)w1P$JSi5cY->~W9JG}` z871ygD_p)tHz#zl2qEkR)Lo1l$;W*yaQZxiwzkHZxG1@!%=T~C0BD4nSwmucl|oPbX0z1;Ld#@o^PAKqz|tFtzkyQVMkd! zMk=9~gjqx|I6Ik=i@j`*^=SWNO4$zCIyb6P>6(y8&cFcVJky{)ZG~^j&g#2pEn&Z8 zez;u)u^f*=K2aqp9Yz+yUJHSVRH&FB5y2u88~D~s`ZyWiP5NeT;U#nAtJ}Vg>Vw*@ zb}6Y#|C0)MlaaU|5Z$9Z;I)YE&E&|=B4pF}fjgr&n;C&fRv;W9G?%12jHG2eHD6zp zmwH#LB~;(UeAu41lFNcHnN_uXIM>=51GOu7ir#I4-fcuz)+3c~vs|kfBM!tVpr5>3 zlRUig+v|%HwLu2I!QCnNOrlRpz10}!Ea&?7G)F~6_2Wm)^D9kOXp1&YQOV(M zwY{{M)bP`6sMNO{>`q9zYqyG4S~my=9d*j}0KQt`-`&S5_pJBasyR?8Ie-`yP_yiL z(4Kuz@p+*7DnL##N!C7~VvE7!MsDpnc|dP_QFHQp?R=b~iRiBy&CAvge<*(3hfJ@P zvZ>LFy{5DE!$wEi#2KLQ)WLO;JITgM41dqBOM8Dc<#-3{NYKSWc`E3ZmpbSmbOt`h ze01WlX_Ph1{Y2OGV6J@(r}fG#r*d;R4YV<*B&kZHE&fSmvaWClE^R>+<$1xS+|&$G2T`Rm-7uRVq38w=2vf2c zr|D~?AsF-|@zSAZR;khYOjTiU;g}-G2E5)a8W3EpbN-r|Y1znaKex(DLANtj=8h$q z(K9<#(x5bumEgL=X8*1C zz=$Q&hXBbA-yuuHb7ekyBMqWI4l$>il41swi1pJ%I70^o0*F$9j;}~%nwzGK0MwfA zuuXuoqsz@o`u?l*UaD3%q%!CO#SEYk<|;n%{IPj13TlZ)vg?WXg_Er`;9fOekoeiE z$vMkNI3VUUZRH^l(#<#i+7)}118H~v6> z$Hl4@_@Y`4zm_L-J4A~Vu{Rob>eUy`SH7N+`Q)bYYkR7DD%p z{CDv$@7J1eC2joBM%&4h%BL5Ac9>lUVI!0cX=DD1h7nI%kENScHDRDX_Nci?0PT#p z@;iK3mjrMc@nf z*B9K}g%h3nu*iU7gAtut%F0(F4}{+$`6rQLR#%}+ZCU#<52-Nys_T+HKhsvR4K^-a z?E_nd(>y9%+MJ^`{Gda_4OO{C4&io{i^)7spGg*c%*4YQ5dhHM{KalkLfKZvH49?b{M2&_@9=D8ViRzi1Aubd+4^#|(iB6Kl6uGBLH zGv9vps_32uwP`?o$@?~Pz6Vj=cZ+v6i!zNZ)roXcYG$*V>32DxSFFp9WHAYm3AC!T z()ilw^V3;`cXna4D6r9GKClsJ1vv{-R9&+EVb*{C%ZfG$(0VGl6s0MxQw)9D9UU+4 ztjbMfC9t_AfG16I1yS(?`(C@VO<4cXk&)XoYopJ%YiU`?3~3Q0kY@)xam<-m3- zY2;&krNzK@n0Zd?<7%g9&uF=f`kLSc^AZ8K>dDR!iInkmQ8!fyn2M|_H>I_}WAo<- z7wz9M;LxPgg6=ng7lHfYd=`osUgq-P%}W9eWUq}8BM9sJTp(0+ z{!WZ8{Gs)rR^^tb==A2%b8w(HgQcN4_vy4pSs1jH;Yp&`B1+xxAmwK$#@C;mAV{>l zL6N*c8qyUy%J6w3mMJPoDGAaaAHHb4k!shGjPlT^DCFU&EWWa0wR_j8gn+rL26K+o zvW1dWm3A?!iz(CyqU7xdeiQ^aOd3*?Hm4AOlW??Kf{N=uPcaX@$ast$Rk(s)hTXD) zut+vKA{)S`Si;GWJEF|o;O>u#$WJKK9t;|p$Ncv3C#-Y=u|jW8klU)jdm-64R_r84 z8g*hJ&=+zl3M3s2n}n(EVdyPJd?+$_9lM6P*UFccy|x6y)x!+xvtOS;hiDuG@T`_x z97P>POP)8{NKC)h13Hc1`@iUPDT{JK(8U^CD>)U>#U3WpD7oySpei{}pamz2tBZ^R zWn)#Yb&yZ$Wn;ri@5aKf!d>EOh46-QjX4T#`1q&_@$J~(f0YHb+0hs<|DYQD*@H$r zXp#gZy^z%w>3MJj5o%Ou>FLBlN^KV?mLhlq;zf_ zhWo(VBk^4IhCxb*P`kU_7HMkel|TQ<^_z;DOogZ3M9ekPl3GXp%rN}5_Bm=q&ADl( zY&R3RYHdQW5JM$%k~OA$D#jQCb>k>~=_y3dHS&a`xb$?QdE&>VblFWwBGAjB=Btqw z)hBdnn7ZGMlt=S!h}xtdC-<2Px~4uEbRA%uCdvv#&|R&KwRcSfwz#S6+V-qq%J-!7 zU~R(g#Km7`nIfQ~tcCO|?A~t{_$>0^B^|aI&rLfQ0n&YXS> zmMm9x^WJIn@M3opxfeqhNN=R1>b!H|ZB5Tq}ZC_|jHQ0vB$<)Xa~d|>37=BqzDLe_1$ z8*L1R3o^BVkPNSO*9l&PEK-F~EkdYHVKraMyhX~)KSAIgc0JMFT(clQ^FOV&hzA72 z;(u7}!qxPc*lVFnlfAhb-uK^O(buKa;HPP)a4py!!3Q2+*u@Xa!u%t1s%Qxcarj*N zMbL6_Acx;L41{Hq+5Tmq?cF`-+5m< z%yC$}+k2q+3`a~Y3pT9q^6{OzqkPZE8Hgmzh+_p9k+KAs3q9!FkZ2}hRBtUT`Z&OU zBt$eWJQNWR(9H&0@1CM_6hZ@Tle7pHQUP-6&l6t490J1D^E=h!jxzYXEpWKf1S;b{ zqhn=A*%WL{v)vV>l9k(*ow6J$I&Y;}^MUxv z5qaqUVz~>EXYx<4T@_DSW`6u1ap3uSrs0Y6#R-n}>QA5Dzc^553e${%&o2t&5ruQ= zM_j~wesE$Q-q|+1@(ttUVOE{x+80aOJ3*1rjzc&j^TOA_R?X1wUNv#NqC>*^Qiv^9 zT%W~fa&QrE11`9qC;ba+OB8&PCUlZm1Y9G#E(IP3Z)~5zep`XRhiuv~f_PChC2cmoMPbWu;i=1;Zuq);H2OhLF zzg1@5`7d}d`pa<2=V`*Q*pFsb;3oBLafYrU%o7Hgl6ih%%?q`gaCjm5YjkutdJjJP zKD^Z*I0cZsL&By1&zgoycwkdhn3qet9JdSS-_53cx=2^6dK55N)hnDj>9SKSsa|JQ zOw9p&PkG%8*jFaStrI zxS!RaU(T3N2++)*hNC(4-@xExKNt*FSx$V!9uzfJP4Kk;1#~x=h*bL1YRvyCx$hqj z8HvG8bt1ndjm@9)tEoihLt{}N-Fw1Q_60OuR7TN-BnL<_$KJ?BjHyhn^G7iF8983! zfSBrWR*;y6=pE6b%)K`@*PWc57c` z-()_~lTQ!Dm>pbc*$5wxdDwhqYtjx~Pn*FVGC$|QW$M>MSKIH&xv>eo22Sj8-YEeF zFYU2jfy=MES+N)O8D&o0;tvIL!m2y=oa2Omd(D%+7l4pF12@pjBmxQ|jo{aH z)=`u{d&+I)^5-r~iRd(&Kho?zo9)dikHI7uMtze$_mhp)_P}5j3pz^Caz0*H&VCO~ zYL!}En8q>q7Bw`>)AldLOMi$;5hx|BQ{#&hGWqoVNO^S8HsmDa;n2U>MdsyF!J{(} zBux&(l_Uf$68F4(q#(mHTv1?d@Gtw|6*vpm{x@h4P<+1`ZG8si^OJn=B&`N=q6vH} zfK(P*xe*kHS!65*-v{385%7=l|C)~4Rd06-M64!sRHSV?J&Yo{9CJ-TJ8Y>R=ynhC z*B6{a)TCIKh$e}q@ebbdlY|+t9hf47n1;RZ-ZS!V7+kS@hq|qy&_?!NEItR^q8t)U zKMU+J6?jxTRSq4lQFW&S9HjxPGldYKo5%}5(-a1TwPuR6)L9?Jb^?CEgq{2ZBqTyQ z5evVedn4aOO$bfk60iZ%{Cg?Xgpjl|LUjk;RyJK}$-Mh8z%|$dPQd@$f@aF_(34wev99g+h%2Zf zRGFSgjQBS(TFEErO6ezc(}-u0S6Mj!Nv)p5FOBMk?{Daz0`C!NJ zc;N?oooS53_SwjSv^tVrgZ7rLwM&;KL|*SrK)(*A=p}biaUxo5*gBdrC(aX-9=%(r z8Z;|MPZ@bLEIK$dBEVlBprpR><;MR0juZja<^4|dgw}v5K}`!M3M!{V%TW{SJX==$ zpbW)bwWzn{j)cIi)CvX>VY=X+Hr=X1Zg79~YKIgMon&KRAE1X$-1Pc39EaP0!aaiu zOS7x6=4)3F%lxnux(ejZI<~y&!X=x4vZ9LRD%-%tyjAW01rP-DAnyu<)-}th7XD82 z16UXB-nL6DLVM>zb~z*zZvOChH)Fxj1rbYR!*{PSqQSQg44nqO&G$2c-nY4C52EKD zO28}d3G%jrqSxY%arI1 zIE3Hn0UwSnWQlYq2n=Ko1pw8dYq)4A*X&Uz01hTGAbzQ0_j(l%uGk-ku*M*^uF}z7oH%!se!%|t z?<#g<`Ft+`i`!|~c7oZ3nf)J)GbxIp{|~m)S`u{opKNFJpKPZQVTeM??myX%J-2|c zvBKmhWzxD20imHA6be>M=$IHd@Hh#F*q@|-K8Jw|Xc8Tv%JULqF>Rr?CViu}9 ztg?+080m)#p?iz6r7>w+B8Q=g0^AZgM2Jtq!Uj!#aDYO0lnUuTqs>{c2GHXUwE@cM3E5d5^_e(Uk8c*w8q zC@8-E(GV2W5IW1eG0+>K3Gi0YHF@!M;`->gn?yX!24kIngc6*{NMS&o%xAIFD*oA+ z7-jTu`V=&}HFA0vvA;Fa?42W2=krLXnL%j!FKx2xcVCV7e%1!6Q0^~r%3I)OV6_Cx zM^T%JD>(MmdQ~qD%f_rT{cmvN_Und=<@f)E)|}ac%KZLYmyD3)E|<(@jf5nH1ub2e zo#dtFF|P7cRC?Rp-MVf+dA7JMmfh8%!wk|p;~Piwmdy_fQ5ARI-T@0 z2jf)A>2zN1OUK@}2--RysA`(>v&`xI^~(@X5f^wS`{=V-*X4rBsz_=Ryp&ElhTk2+ zc#V5YBu?A{FWoN}fQ5;lhwcW4uS^U#2;V$2;zD?lSHDVy7KgJ*cSx8EoBn`D(D{eIzCl0r zfU>|G9dkE>W$O7q1S1mr{?BNV?DMsC@~zjedKUR}|E3o^xc*fy(%Qn7 zUJJ-dJ$Z#unKiYBm949|DJ0Br)#FkP8{Gcwl5xxx7Fa$c=Kr%x=HvAtuLkSi1fxcL zsbXVLb^(~#c+4g7a|i#lhHm;}l+?RDub3?8T(2Ccw?TGjx9XJ~gWg_Z9K2qc*C1+3 zmuyuQ&H7O+pCa*rHNgmrtGgVTy;dj#BA54ZadZ8yV8pX$R+^DUo-lm3@1ETJ2vgZkf2`qmwI>xqWD~qG%PXS^nJE= ziv#PXIbjh&_6m2Ey7hdpP#VsG#OnHOdmm?jmG#9Hr^?4GCi!|KInl+6kAdK*941?m zMns%0EfR-t~ygU*jz6jmP%+!=g!s3scL~8|?VL`DsF&Vr0!Bao! zlS-+TE{-Sp?XV_f%A-uyHgfWm&(;*_6ln9NT>R+4zp<%^o#Ur8yDf*_{g1)I1I^Xt z(eYnd`Fxgadj*!2vwEsvS*i4|vQiB!D~U@I8V#H*V*e>C6aU&NGLEAHcF?(cIAc;p^IxZt z&{K&tZzCoHjEv;>H-E)1yQ0A<{CFO1^$Ehad<#?mG52clU|A+}5AB!CTtk6YjgQF) znd%b44C5_~UrLhNTOVyookd zaMC!R?bd9f=C(wiHg;L5&x$cywRKEUtoedpY?H zlJS2LH2z)@qkkr!J$^ZhrkfVJ1Uc@aM@Jl$zYr@&GD!M69LhEY1gz(1l^}6Q5Brs* z4EFMN$<^oy;F!wh)WNuDvV_)xlcPzwN z|6ng=&9DQpF{>`K>qERNR#y^{9*U@C56-DRC=ju@LCGN!z!cMvtP6);k=2#k3b`(& z^b?b0&Wv@WUo}P5q}l6Dq-JLOj;n#Q*9zT^q<;2O87ugYwZJNXHkpV9-8BL&whaau zFXuK3d5xey?+-$JQ4~C#pWwwEcAd$tGMXiht|J#}+b`4M8R+3c^rmSKzHcH6PuWW) z$D{zb4p=x{^I$+6kT`3>0Fu}TIYSQ+V+V^5`Atby6R$J!!3cA>7=PJE_od=9-dJ7s7{VABN{tz-6Unz+&uA^?~{0OsEZ(D8_*X;ywBOo3mH z*-JWtzvO;4?)V7ckt_l@B3J`^%lUiDE#uo@Tg%H@4Jo`1NkKOs*`!DthxP0 z687g}@!?Z-#{^%i6q7Oq1X6`z%&WTkJ~S*Tc8NTeOzHqJgpYs46BIko5ZNCp-}n~} zI5?v)nG!n(ah_r|{=5sbo*9A&tpow0JWyz%OQ$KuFWHZEv`s42S94t!@RwMd=uG z4fQv&$P<1oB`*Cx@kkLcVr}3RM+zhwz83Asf!fj$1}j;+w2oo^XALglvoZ6N1z0fwb`_3KXnbl8_|U$0}ufGisHZ6U5N3LG
q`>m;rF(AAk> znldP^CV~aCOYWbcL(AfW|5-7*<3(R?&-T|@eW6w?ce=-#B}uyr5ld*=izQa`k?M&G z#9>>ic2H58!WJpCk2UQi*c!A zD~Wnq*KQ+yTvP6|dQ?tQ*(|rC_1I|hE5MBm_VwsfS2*kLx*JQw)Mw~)%i585G}*IK zBR46**>kP?!!lv=fw%lHr`i})C!aqju4n$;W9n41uFDbES8MBMc97`ig&i-#-vzz5 z(zP-B32KJRE_uv$G|2H2@;LWBL-FjFc+!ysC(iy^3HyV?!yhZWzUJ=UL=CS<|07bq(m%1e9czw?`7$AP%EJvi z(vO4;{}o$~>z|RZ{H+@zCq}BDKOV|xnA#&H8$US3wPowi|EP3>^z11^L`MUk2@Vq| z{(aSPkqKUPwEuGTn*LvkQxA3jYmocH?_Yyl_0Eis(F;Soi1jf4Ty%i4|NEljZ-1&_ z`G{C7;S%%V4|(a~ee=uSWVDdC;upPzTV6Bc(CFIWAg2LWH>gUCq$5_~TDxCevBlFqh-`h9^-g3WrC8btI z^wUp^X8!i1rW9N13NGZkvH#JNI$K*WNEH1l%u#U3sw7-93E+V98w(Y2l<;b<{E9n#+4g zIy3lp$w3%~P0>vFt0@)FYo0`B)3Fu~NYk>Ks%0A!In_;V%YdwE2ThTPC9xsl_M%Qk zoYuo35Z2x0dF0?W4+vKLsEX^FxCKWM+IqLiAVLdt{&@7 zF6-LwF}9qi+GJOLuo#c72Vc@8FdC1p)hRr29$Ju~{)%BpTAp&sj*VH3$A zpZw=fbk^W}K3AH$YW^6zUJpS^x>IF_gWP0uCc5?|w>(OZ=q)whpbRNYDlS`smZj?4 z;rP2hTW;xl26DrF8?`{r3|Hx38h|TTTQAJ4s>8an<}{gO|9D+>v?w!EB@PE9=E@vb zjIG#zAMJa@R=KNVz_p%&V^SW|zRIQ7N0oE47gNe8^6ZSJmZW0sR%6V^#EQp^o>9#g zHTHRclFG>BgHGDjucDbw%^$dFD^c{>3w#|vj(6_d4JW;Gnb-^n%=IScwMGZiBiZ>% zd)R!oub(yKdm3i_oeC{%kEXTAA3tSYjH;WL>F%~?Q17wM0^-g6mp_QmJ=tjo7A3C( z$tH0`WZWcV z<(!xSs)N;m2FB#Q%qwJzkp81h5D#XmfQ-OjRDM z2H51bu_edE1L<#iAipfZeW`ny>wf$#^`&OF<2lO*v=!3*JPK@rM-JuQ|3mbGZcJ4s zem`m66A;*lMgbO=yRqF`XbceKuxvQ{;5@tWRl6Zse0yUdQe{8bY3z@E0}S1rM7p92 z`K^s3Wl7C%g~6@4?8wB)AG1x^oGrXkm>z**E3l+JNh10+OF4npW}g^ZPUV23kck1L ziHkoK7)`uRfyuBmamOfoD+Hm}HeMc*GzORUf!81?rv7CCM<7iPBPLMEpPvdM&lh4Z z3|fgeuQSoL1HdHmn4tj}yWmOgX~A^r&?FkK6d@9MY=cmG11fc&v($@lq4>r@oAxqJ z>GQ!yRyCR{YjvA6iAHkX$i-ODpcK**I-!M_6i_q3Tzs`T-a+i!%tml#S*`--@d|oU zXy?JCNWm-+qMrEz)49eGU)CF0;Uf1~J3dDUlyauFjM*t`oT)EsYYK7D5#` z{kQ=P6#|4Zb<|m)gy+91!44ForeF`z3Z1Uq#UNL^HVc|jN*I^uTJhjl6zd)g{YK{e z{Nqo46qrTM9G|U{gC6O~$ zo|~4s%wC}-Q8P<4b|VVtr)K5|FJt%eVfA@lN!iV6kba@z`SiUav87n*T&M^GgQXaP zOpBZ_ATvHgLj)5dGu~J}a^ZQ~6kZYzY9N@LLK?^l(EjbqVoT?&`zLd--aTK~Ip(6X zHYLvoM_!U)H`uo$>Dq3xNGRUHLI?PWv%6yasNry=E7c25$f1EAZysolB&3^69kmu3 z)M+q5`r-{XH8qHQ=USV&O6Y2eoDKR*{Kw-*9}zd@zM)X=;IXeaM_WF!dA4hQqM|Vu zljM*Gn7U((oXR%m>i8Wj1g8s1Mnv~!qT#-ad5Q>fQQzYSwMKam5ZsKfB7J~}I{@*D zqb5~@{u|#+X z!WBmxr6@i?Jg)OJP*S03{iFeABrj4-IOlB=pJlWf%?ngn`KrDyegmhdz6*c)8*=zT zqUu~-V(uRPpDfBA-DdddStUiXKcF1mTdVaBeB*%s))dD-d9U2CCct~vg4&UyW!L>n zg`ov0SsL3jnQa{2?pl`IeUdEoy8cR!d-iRY8|Awl>Rzj%teq39dze3Jz0# z+(lPaxn=87xr5k2FI;>!{qK@@-t^zNy}6C)#6P(`we0^jxBtJX%2zD@OI6VIeVHvfOvJ*?ONxhoHG|5xjc z(A};7L+Vlf?^5rP0B`J{Rry(PRUQIwOwuTiIU#a@!F4|6z36<{k)z;y!VJrHi^-4x z*<1;9DR|Z`ENm zDq7TX-~{oO5R+~=o|y0bcsq|~lsR%owF){qq^{d71!Sqn-lofzPtl4x^MzvkQOTR1 zW+OZJQ;?m-D`q0p;&l;_n}*WS$$qaiWJYM^cqdo|xEzZ253pYtMwEX?0Gm#X2& zqyNr!wRODkR?buVGs-yS^n>KEN)%IUIO&61R>`}z)#5-j7TnD-$ei=w^wt_1kq&0R z9=(fF_#=JO;gr%b!!vLH-aJITN?ne>ESq}M5Y^nF$n3J+X(Ep52<3q?G%?#4MwYjf zrnT-A4gJy|_G1;@=j8f{{aoc#9YnS<1g?kZ=0TkAI7BDMb~o&%?ZM3 zEX8>H7KIgf8UlU_G_4NsF_k?$ZR|oXsV{w=xihKwGwfr>!#ne3`kfX9cr#wzheMhU z!HRNWT@~%ab{fU2);x0hKZkOZHS63IqsDF5@ys&L_!%=G?(< z&}N~52#`jk06|Fa#o7YA0M~$B;0U`N-G!eI*jn|1PJ{MXgJ?hc#?z6^(JMkkg)+n? zZEC=K>F42$EhkRXCXx*p1`-W7{7|D$Z3c>z+l$b4{C=ANr(dgw;MlEE208GPI)Ehw z1JaPL7^YV(-wG-ao@}zX01pVD0(=IfCqe|$+EmaPI+w`-nSft16Zbb~L5aKo2A~!l z9U#@b49Jj@1gVvIfe=W^Ioj2$g4(MBqzt44$hAQj(%R5nDB<>HtHL||V;y%H+ge-0 z;zlu)0H3EAGVKRYkX=rZ2da5PC>`ZwD_!_Ya!-=gUK4pf`d@!o`Kl_UZ68wdA%JZ!iZIALelnANHicG zlayLS;S+E-nX{>gpLmn1SDwA7p%wn+8o&v(hL|s^2LJnZr}@T9HMdtNYN5VN8}sdr z9?a0;@HdQp)b%EE)UAnOeTpofEh43({ddMbcZO>mWxt{T*|E`?(>EP|xeb*iSpJB3 zKgZllAV7T)5}pUK=0zO9(?a5SOoa)?3DVWUjv{3kJ{in(J+1EiXg@) z3|=%6CoN$Gmx>;u@Zc1|MWQ%N`!77T1NQ_k^RFu|-dESm{8HcN4ET5!|A3P9n8;=a zTMnCkw;UFmL0{hH%I%gcvDde05}zA6)ky8aHQ;UxZM5 z)z)O>gRJnF1c6_4MA}|kv5@k=g(||Q)huAp)pqDb?WU~F2PiY;7ns6+R6V^vsqJ}N z_5{^9`Nwv@-p{)!hRC+g42z-tO7(J}8&WK@!_+X=_Wkjaj6Ci$RdXw;;YvY?SXHoV zTwJOZWgB}-nAOyhm{Aucx%TijN{3O633>5#e>ak^oAU(RZw)hJ)ny+j8saUD4Y9q* zB--Mg~`Za-^=2XBCe5b zse6-kwXnfmYti2u5cjwWAdPE?cVmy=duK#t>UKQrdU9}_`FAKXByA3We>>N%oZrek zj~u&QZmlj)>Nhb~K5$kU?~T{1f02~S`0X8B*_ue6!_@OZvm-j~OB+b{|Do+Ifa++s zb=`@(yGxMZ5-hj}cXxMpXJSDD1c%_kU4sXL1$TFMmjoxA=Kt5)d#%0qx%Zx`d#Ivn zcGDlTdv?)`_Z`m|gTb;!BZ(zLo5A`3l`&QHibR`vzeX>KMU@%v7Y<|E>kd}Oqb8%P zO}5W1b|xPzwb;XU`0C6$O4k#-pWe|#B7(7s()>Q83@<-0rOIcr3KcBjXt9Ou=zeV} zFYNXoEF4ZT?mN|K`|LZ@Wx_-?UaMMu?`}Jti4Cu?=rxo_chKpeoUxS zUI%JRn549G9&4$sGUG0nEjT3%ys|=GLej;PSE4jGm$P2H5+O5wSn{WbC&W1}lq7iW zu2~vF!qeJt%u+4633NxbEk?^dRM`_xn(WnfCVwjQ zA{yxFCYtQwN_{2?&drn=)Y%|a?2o4 zbgHV8ch)E2L-Na(QgpgvL1c#&Lxy_KagWqdr%B8t4Z;C-kN~>p-&B)_|1%M^URZI!}KT?m11g2#8s|CMpxAAT>E6LikmTaQ9cjX0>@5 zS81kgGV)EO4q(TsZeyHNNx>3x_15<-mk)UC;wN+O_mwI}{C0LBt;1yoJxel zfTEmE6g4&T{#2xS4-0jEMuiGf|JPb$xi1N{ho})Pv~|i~uyE>XI#W42}{0O7ifl zY5pybhI&^w9p!p_m}(FFhd?MKS^-z+3KA_oV7}Q(@6s0`w~6<+lvapHC2@WqaFs}) z$eOkx-13n;t^7(R$cPz!Pp1T;OE3L~Jk1s^xaDns@D9Hi0R{k# z5ntfX^#){IG9|!NR>3Y%RQ}=!>7{o>>O^x*Zlp;&31~Jw z77JDx|3YRG%Q)(QOd*1fSwM%5$svY9A+o!Jjv3m>OZE0W0Xeo5@OTvqlBq_Ymn0Mp z9ryTf*GaMkJYea%i%~rz9NV3ts|Xc`RbiTjp;ZLvjF3E|=i$pbZZ{l-7WwnjHZ_bn{^!^Dsj#C zr3f$TrAr9c*N1{F)L|y<5ZH1TRe%|vV2gbz&}bFhC@DlbaONCHR$Q8(QQ*uOAIUlw zzwxW{_g&BZ#M^49V5Tz?@&%wKqy}sR(FGZtMGFDl#s<9%2dyiN0FR4qR|N_NF6~&N z24p;aHA7L+{Ri=_hCKXR4Rz4Z6m+O4rb1Y3T{7&p;yZ)?$ch-ok9*j@WC^RXd_vdL<*EM8d9mHw)P&-1)nag9uitFhAM z;vtaeR=U(Ma$={MgjI6u3G@)(7qkjeV#|b?jv?7kHT%dB(u}RDw)Js$!ARR)a)=wI zu1Vw{DH8JE^vHX!*^$u#xioOfZ|B^Tb8=j>n=h7$LA&80U@g*mEZRA!q+faPaJy;W zte#}Pa9|jGO4Hk@-D%w@{1}XI28AwB^5>bEVRXgcN_D9l+Kb zB|71a`Zo&yj5x#o`cAve(u42Ahw#<0&3E;#ZVL`VVtb~@!uaqbIbYBM=AZg!3&e+; z{zj?>B<-+CtPdPv-aR$l?MI>qI4#*Y`>hhIvLS>U3pekwiKJQSWG|=7Ornf`*17Rw z=oX1NL}U6FAm78*?Bs7@zyE&@alr?4lm-}@(% z|9uu<@*7NZcl7@gtID4S{apgD;rRbr0>8gVdPPI=y=}QiP=1L|Li4m;cIZUaZLjt} zKq{y1L(+6Yc!kMR`+M1R0{>sx&8-!Ou0y@>1cop%)sRr9w5*gL50R!9Ug3M{z+X== z?1(90K^^2>V*d@LA_Gz>rC136!rqhogQeR3HFVH{8gUL>Q#o^=8ZCieKkq}3h|x)Bzi6O%(+B!XFGx=*l8&CA;G!{3(ME- z?cq}NGoKqX>0^Y3)F3_y~p{)>PEJYXOjPsMzg2XsHsN+s^2)NSqEvcYBkz0 z8k{k>-%LZyn=tf@Tok+T4{1*H{$ijYfhUU+l0^0A9q?i*>J(H!=OCGJDE`hc)8|B^ z!={SGKtSy}eB5nQR@o#)rOk{k@H75Se}g4letE|lI~szedMs3{u^;=2vgW|JTalY{ zePc1U8iH*d^aEx{{EeF1-U?rwDJ6fnIV-8;gS_)RC^<5W!GP;H3ALpRht36VdAvt*vM;v|3W4>>5|}HrH{RCloaG_ z!Y1I}1V|}HTl&wvMeqZ!I0h^bFcjGF!%MkLpC^D0$ZaoKScC{LsNFBqdSA51NK<~j zJ%a&N6+<+y4NAwb&F2`sK+eFFASr2DC)*P#3G4I4wEBQs5#K_%LmUm^2c+NW zw&597-Cy;x;w`j*oFTR#d(}0}mtozuBWyPhJ>*~}X%wUO{uQ@gKTck_Afsi$|Z zvoD||l;QeFk2w&!WJdU1zxLT~yrjl%f?rHi#NwS&z-Yo`w_RVH)nLn0QQ*|S(-%H!yaBVF_qfUT;gg1j>L$}sI^b_=U#63OYagH@oqK#K zrD)H72Y-=fyW3MY_fCr$XW+P5&!ERexkIi+Z$h)i2oZ?-yVR~o7phLcxtF2w5%%{NKuaraAdd zPDjOTzo+7a?y{e+?v0Ev`n6qlA7;Ir+i*{}vz==+kOp0&XKVsimpEu)f>pT;IB8d9 zN6{2Ar`}7fp=x{`+U~s=;|o(i)Mh?wg8ZHD`%w8(ho9z?pHSwErTXLYbi?_K0TP!3 z0LB~bZ@ryd+|s=A6A%$^NR5Q$3*p#|s0&efI$p4SAkU+=`T7|nn%fbpEf0V!;L46V z%*u{zWS4wL65+}Y_ZE$z8$CF~&wq7C_FW0FEqte9K-=>UMaU2(g_+Xi^tsAh%FDGICW0oF&E ziBxHw6g-`JR4GMaR8fUet)5FJv)an!LJUcybcL}-GQPIJl}k`BeIcODO>EM%%<~yw zDc3oHEerCdsI%m22Dqv6BtYwxl^Xjb$7FKE85bt`eb7KQ)0%8oDTNe%Bohp%Ff$4- zyiQ6T?*~fl)wjCvJhC+rUZzd5^`AAz9y|Bt}`PbfxSE?9U1i+$+RI}A@bLxH_ z=R2~geKQuqiL4!XDwKU})at@a2|ibWkcCT8h%ucfb3OFy3?_F>BIV`s`E1IG!W7xNAN3&`8|I+Tqi?A?c~s<6+eLOKwhi{ESeN6V3++ z%Z$AqO{*v(xM$)W`Pa}9(C)MKnR57hNoD11UptcX3DRBxQ^Yw1PJ0k0n;TMJ(s!agtGesaI_p+-5(nMXxEV(+udaM zIZS7-{<0t{zTNSh&PPg&3*|(pBggMQwlaJ?M}K-F11&O@!l+EhFD-{6jGjiY@h**k z%JS^T4^Z(K6Y6_L6DW-Hu^_Y*hEt0R<1f4yl%cTEtkvH#CW8tQA$TY>t+`EM%FCb` z*q15#P|NHgd}Y6_^e zB39+5AzR51-N7?492|^N^H2nM1%+UzoKw`>$uW}9M-IW2;!vC;iN}+35d`;vx9LYV zU{9F4-+%tRoSnZoa$W1m{&@_n*DjP#w%a3{gz%pw88v`f=EYFwgR$B6pdt2Ma)eDQ zqgO;$HT0d$)Ne#qg^-$6hLVDbYZZqicNZX6&zJ`Z(}UGOrOFU->0GlM+|0tIZJ$(Z z%~?QcfTRPLS|5W{A&MXY#;H{flGWXmRlP|cHVu&#$H+zxR8%C-$vgz@bab{g16P0y z^$l1eTyA{~)_|yl)Pea1EU|JowQ4}KY5-$S(C5y|L3~T+g0bQ>M3}H~CQ=O6INm*k z>3r3W6~G&5LDWDR!E^$v!0x7CcMup@1!4fr$w71i=e7{757P4_YZlx{Ko+k4*$(w9 zy%F^QXM!PDQ@DDV3T(jr(ZD?p;_q!|h+n{xm0#T;Lr<~}!ILOn;Tm8GbVeg8peh;3 z7gj}U;G)hs{aj^;pNKXCe&H;d3ijZ6PVzB*zYrDMkT&4M=kDSJIy!GP5kG_h9i1gb zh{yYa>!cOl5OMtuQ#me?coS03(Ww>3qGH0x-WOE4N+(;GUa0OG*9v1Qvzwrl%SI1x zQm2GpcrfqO9(T&|@WUtsXzh$1fY;HpT_x6=1TA5+xM4=@@G(j~sB19PZ*m1co_X>M z+7QOy&C~2(aL0k>X|`+<)D9*6uoI6yN?SZuL*Hd*Nc5UZmf=ol77+H znt>x086MWvQMcXP%@&^f&ujfn{$9*CG-gg>MME14l^*_W@dhSVAN}3!jtA#)WdQ+Y z!P8Q*+cp~;oHg#)WJTa)eD3|d8B4_^wUvGKUh8kvU+ZtSgC1&!YD_cfa&Fp&Ukh+v zJPM6x@0{FgYx)&I{p-{A0!O(>^Ouk!6V-!E>m?xp(rU20e)~EB8qWAZ`~hqR*&B(| z+h>ci(}X!$z!z4u??N4{|1^~aOF2Z$I30c@vR{43<0CHRh-m%dySJl^#P3{XyoQ|# zaF~njrXqvLFZr-y`_ndK!=4WtKDF#N`JAp99rAkb=@j^fle8U>tU5IL^5PE!wymCR z{b*maJ}$r*+2E4*h1pO@Hc zei$k2nCO8^sApaBhg{gjlW}p+rM<7{PmX3(zi#puE8Qc%8xWbfAG3`Avfe>wUXVC@F~*Crmc;t zNaQ+dvX9(MfSBro$G3BveH|GEH038|Udc4=g9+L(%kCa^2tY>uXh(!;TD5mak?=Iw zIe76|6LidJ8k?PwcHGjo#rjxnZ?c=+pX2qV>@%53_u>i2m&3s^Lhu&V;cflLgKl$E zUhhxsP~q^zkP({@>5l&42Y))As85pw{S?&z3XsR#Bs$vhX^y)i2-VaNgJ8gAdVv(K zk%694C7;}0pq?HiAi723NQ zN}9dXmTY@Vo4o1>_D`%qu*&6_{C8q4@cpiT{!)=MqFU7&SrqI(k`LWhqp5)=*OA0C-o(W%a~2WgajQq>~7Xh<@}fdSg0;R+tkpA%@A z%J2wLeua7r9*HhYSA%Imib-{20gvMP-ydrXQ_Dj3RYUl52#?F4IuvN+vTGGb{#5`_osy(61NgX@$hU_jtU! z-0q)SZmJxX{WYy9)=%(|?35o42!dH}$GR^$D#Mf&_~^OGCwWuGrNLjObS#>gf2wFW zW0}&(Y7cUe94>O%J5o=ZVkGj$kIPaElNnzi4slX{j;%3RZVnj5&LK?AT;i%*(bcd{ zm0cW#b4c}CF5Dr&O@@1SFutw{XaWsz0iX#&e`{~(;rXO9&%oo43WJN2k_`vRATApQ znWm%2ncEndt3_V(J5fjvu+T+ueu~h0dESs%Z=mn@Rij3QO2mv~CgaW{2lTiU&9Vbx zB12&`@KNrwBZkpNu~OwLSkLwJ9U_uQ&CF@df0lDSE>qJewv~b1uoEotOQ-)-)mtlE z!4%uo*(mtL8Tya-hM1MLtN&;otM7!MG&lS&arV>hDVj7le`6aYV+LEclz}b1>Datd zQdL4M? z)mKylC-iSq931T2LMyVh^vL;Jll&B^ih?#LiIVXn45xwHG9n!M?JRjX0ZL?wddHWN z69qY8Z9oRfd8!cDVne5`^evo%2BQZ0zjRE|@kNX=e;WHdij@_^$W=R`m*h=afC64P zsayCCQLEfhA#As{Zh}?-#PW&-d$e(cyTjy(C{hS!b(7zb*%xO_s0PgH7Yb<}2dYOT z0>ZbtGO{LVEFuWyK#3ht()M#ja7R>4+8Y8Jb>GGm%<3S1)};>Gag}PRWy0~N0D(Mg z2Fexwt?}}M6HSTRT-~!Ac|J=J3h4-J@)&Ak4$7162?QR08WR0tH^oK9x`<9a5XIZh z%U?YX?zN!y?=x# z7AC+HO6=ZCJI1#eJJ|Kcs)lcqKkEpxQ;eiXxQcX0)z%K+!!>fjs~^oVR;I@QzP2j8 z=10f|Wf8)}-`#If7`XLVb_7P*m1_Li{=mckNGz^eagRt#$URVEUa1e$Ei~~5d~FIt z4&6?-!ZAOk*uPzE`^P3Pz}43JL`Ux|CcssaK}o)u5nKtl+SYKm_iM}u;D3!$YW@+W zJddm_NiHr3h+YnV0#DI!B zkGUjO4xP2##2_4ugl$mzcrS--rCwL|@D*2C4z_pO%xo zWunnE@sNlfp>3W7R)_OZTb)Q)(4n6R$pXX11Du(HQP*2ia%Kr6$*~9s`d68`3}Y*g zFMt1D+-MDq?cYvi@oi5vGx|MZ`c8M(6;Y7Vb^T=HPe&Mh{e;~O@(>r$c;-Fz_ub}b z#8g$T!`_zOLD@CMRIjyoCfF!*dMg#h#I zdbNwF8@@kS>#bh%L{kgFAFc;u`p|teK9X3+Wi0xlF82*AhgLasnx&#+w>_h8yFSJl z%;%vQDZu!F*pQw|nf|Wm*hm%?N!!9`59z)n<(IrJT%jhcugboyDM>y~esT1I{66mBO^UBWYb+Kk{TZ*S z2_KVdRLcclFxHeat9jO4*OX_2*~ii`>$#S6-a~nQTl?;d2KkDW>pLtK)zeEcIi3s>qtAV(D3kkZT`1;r z==#leavVjQge!fyq}iSiyFuTsCMK3oVWyK7HxCq=TccUEkHzaIASfDz%}d}<3mr#qc7FU z=Y?8V!wuC>X=zRSI*nPSksXvnjNQcaS$0Ma8es5*GtZDW|L5KTkY=R$-_eT7MX3skMNG9Te$%&d?q=VNb$GEq z6O}_`IzG=~TnN-u=&MK2c}~v~*%!44;-4avM=ux*xJeAO4#$N^(Z&?%-)r9Q2RFg9n8_V3D*Nl#vX>ej>P22;XYl)I?eK6lu0 z+HcL$ZhHff_cpwD6^`Jw9)qXuNZU(C(zqJe)4&f_YR3t4cI%v`9hom5Y)ct`_?c|x z&+2vSN19u(2&yA4uAQvRYpZ!$mX#ZXe;32j_y77)HjmWh#aY|oI?Dz7qwd$v-*|d~ zvgLNm;NWA&?2@yXJ}2rK(sVcXT<+rMCsyOB}v=-&AS3 zZPwfLDk?xUQ*-l1>jP^RH6*#;F{#sdWJ@J&+24H# z6th6R(9^Yntm=Rj`ydLXX~;lmJvKY9KK5~1$2VOdVE$ag>a2ZU(YH(VErZjcI~@=N z$wINTuVprO$`=q`31HwiU+UZr!PEvFSIG|CiklI7t^S^_W^sX`}_e$SgJ*x!M zz&L=T(ND#%M@6$s9q6o#in`g*kZ4w~bx}Gr?H*J+bNP%yzxuVDA>IFZtBevgh{R*Z zq_3eYfjDQTr4mDn)6m+{(G%KYKy&HXm#j&_5h5Q~V<~<34O!!?qxhZB3YLZx%55m; zz?Y8N*!2zR>U>zmhZwIZp^Jo?pNJdcKhza|6%Ine5<*_&tvj&zNa4lNuB1Lymp+t4 z{LE&2Zyl33=C6p4JsYh+wr_g8n_G4L@Y$lCSn%R+?KHq%rRc8+Mjz=byXv(tE!|1C+J&yK zjU^oEQ*_{BR{m}D-l1x@dBmxrelBR!s|}?;E`VAQ{!kO}HlG1+b7Dmg<9#8{EytAE ze12_uMi8~Ah9V<#TqxjIz6L|mT3qyO=+xB|>sVic_!(O@nPpK6Oz42kQfnjnl$Z*R zxR}2Vg{`YSFev4K$n7y)|53fe4V70voOf+{EpGYdlRP5_-a;lOdWH#b8#B1cxrS14 zB1Ru|Pl9H!6VD@Oo}g9o{Rno3p|peiyy35QakwGxuTq9Jc88Z+(7%w#tW2GVdov(s zgJKtM^t^2~-|(U+)6%X%YC^u8DnouxKFnWvfI+T%mcGpP$f<{nDcsVT&?J)cZ!+-M z`7?x%{(OiNrR8*pL6s+ow4!_nZ_Uu$)4S(p{(dk>=$E_7pAt8XI-I9XAUa#vj$F*2 z`U(%0ihvq$)}80j4UCUe*{H^s{vz(YjVu@GNreX{6{ls>S;)K<^8Rb+3ReJKH71*#k$ zy$F)UU~>L``~;8cTQ!@j7MpErFOAtx64w4h@+l19Y9t<<6l++_{A|(F{ZB z=47AD10y|fK&G=XC-l(*Q1}3|z{Lfxz zX6i6t0LTCVKnAD|WR@xg8mNL9=zt~%)(Qh_%^(JVwd1A&j7@`JGX_p5sm8uYMqKC? zQ|z1(odf+|8?YrF0>pS3;>IGTNoRfG4A>H4oH-tpt&fx~0LKsf8ZR!U*L{1#WoQ-< z6Og(3EV=TrRxn0Ol%k#A${^DkL-ASd={SvaI={(Mfvx9*t$$;KY#?SRK@A|AWOMoO zZ@8X4V~~xNGss5E{bX}_!x6in4Q5QpPIVkmGj%V>NwySX1F}<}5(3;t3NFX^0BQy~ zHA!^FMmo^b=292+S@f@Y1+kWT}{VeO;Fca;tV*VkOdN2aF3y;&I&APhK82HIj>Rm)bS{^3% zjA=+~Uu>u&qQ@$%c@c@$j&B=uwx%&FZ4C^QHH~ri9c9{Il*Zeofsylf`F1CyU+;D$ zIxTU&6P#;*Izr-)AK80(cj!owNV`-s#~Ru&lo7j2OIQz?*Ga3Xj5kwG4WDMf<|xyt z!q-V7F6X^F&FH$16+s$dP_sT8=%I(fe;R?p-3q%H1o=@j0D)O7zE!6Tg3HyQaM(n}XV-YyP~t;8{5t16oHk z*6wungY%E_A`ODIXrN(Yc0-9>fFY^uQJk{I6X|PDMTJ>le8HVpZa{?!Y2TMC>mO}T zuPqfFHA(I9$*P!fJ3WG*Ub44rXGHL-wqb&E4Yy?jdo0g!zolagOe=5y+A;r(tS|`9 zOtYN#n)ta|Kiy^h@)R09UiJBLkv>ONOxR2>hh$~F3EdbseKcESF|V8LU2$j=ajNpB z%MeZ0LY?T-Sm-vJMYE}KpwFp^!+oVe8~4G=_g}h$Op^h}bGzJ)pDJ%iJnUxwOMt5g z+n(4}zm~f!Bq$(oNS4EYOqzq*`fOS9hz*5)2TWaC{qV!~ew$E5wB^Ll^Wr0GcP%+C z;ym52N}rEEVgj-dH}88C!nc=iLZhsmNbW*2n6W}58xMK2T9iwG0JjGSaQV{y4siWt z{}JFi*cPt*9pLu=Ukz|~-n&^wjb?DN%<@C!3}B)2g%LNip& zK$Lm%D9JU=yCF)SYRVT-Pa8?~nB>xWqZoHV8``AAM+>_S;_iG9wl)pqU+=zIw|GnYSceFm! z2N(jTH$DOuBREst=V)kW=kZQLCjDdn9~GHQOxCiGmpzJT9-`jX!(&;YH_$@_3nekw z@t^-S)j)zN^ta6{0cE~r0ZTNl+db_K|ND-?EoF|}OQXES3Y;Iz#GeziBLq1r8hM}* zT(K)Yyj+R(JuF2%@U3T=^{KgfC>b|NFcPk&xB0D@$l+r|C8i%l-_r~5jc;fpIR72u z>UpSDNiS~4y^qjcnkHIlA9W1r;;uFBh$Mt`;(-nqbK;3c;vp`QHzz?As=dH)Wn!1{ zl$^>;dmu;MUyJxRQ_g$jFH;WlD4nb)H85PyBTbP1O7B==rWnB!%F!&bmPZcv>`l^> z;)7RvS`)ua;Dw%T+LHL{@&y!6{9|Z(d&UpOCyM*Po0b#tJJY-Br_2_yU0PV?rgo?q0FNf4Y09$N%o`Ez3y8)?B@sd&otv z=3c+z|6uO%ma+U7bC1B|{~vP?F~11+;8s^-sItn6PrPuome;(NLKbKtV*5OAIil-& z{Zt-Z`iEsxF%s*;RyPINm_N@RDe)EDc!PleS&W>?GADHtPq5?HJ`aS`=_+UP@~|BO z$)j&Sj_uMD{krHfD$)~4&Gz7g8dq;j6CKWR%9Qqy7!}zVn5`I`z00TGDfCTIbe5QT zRvj$fSXE|N-X!tuMQ2qMydlkKqAJf|rWL&4xcG%mkR08Uv2CriQEqvIT7s-Z@L7o< zd1QOJ1$k)-5{?*!xV>{SxnjNs{tUBTQx96i7~GvUFO4Hf{V34O^Zd>UW-S$;Huejd zi!Y9WB8qciC^|+VJvxTE3^Imt;S4%PLwrO@o;n0&UOEwmdU|R>H3c}Xxj=iWy3Lc5u;3|50FbD_u%s`xMXEuDeWN`rCE)f5fOynbWb#R(Qgm!4!Pq_^A z`UN-bu(Y4u%$0%`m%xKcY^atz+%;8AvKMfKCHWya;#q9mZjDOE_XAV(R9?!jca++?%ReiAbvz2B=Pe!s;)o?~cLr1J20bQ|BnUecyLLUxHdAZ{ zI$IXVvCjF<`D@-~5;;wIeJE2X)BPhmPhowgFnfs7zbMLH%Ua-BCGgJsv$o^3Q zvcX`-k9CpQNMM{@eCX-?sRQ~c{ABDhZYJDhEU!(3hu@V|zn3{!_=Ec!6cjhc_D|1h zVxY@U%OpSGJ!|(7Jm~_Qm(NYYu{PvEp3T1gp2j{|5%XEW2Tl;aA;Pz|XjO)A11NDs zmC~gUa&md*ME1;HlFyx*~*v-X|d$1$zxFh-unyBD2fbTv~(1n{o$t4oz z2A`Q@f>tviS6Lw3@RT4AU?;NZK&{h7Dibk6iW`|ktW6yRoh!3!x3@V6{aC@wn9k`T z=W(WMEpx1+k+4>N@#quHWgtQp3Rn&U85Aqxi91Y$+{=dCOIKlFG90)MBE}Li_wrk* zj1_TgCqyh1u^}AsxS1BLc!v~xgq}JmJ{28`lBEz<95uo(5*qzp2fZZ)lKj1}3wXyF zLModJX3-oX5sJo8^{vxPavLj1g(QQApBSoL3eBTcXJVM{@#-axxr_T zSk(!7?S4Oc(6&~11M2+b3kTx2Cjs$*oIo3(&Qxg-*by0g312;T|Y3+J{V3CnFV71TdsanBk~<( z#JbA?-VpLG5sWiG5z-LN`xv{089bTFeB2mhyO|9?zy*)|%LVt?W=Xb5(HIpu!i}a& zmK6=^DCU&WLNTPEy0lfwcEG-r$ zJt)5<(k$|SZ-d*WN#7M2&D$$kECoq)G|5Jzs^gC!*onxdOQ*5+aUO3AFRm+pi+?ZY zjB9Rs3;&ZPuvT&{`nE#iH%zva#r4aVUrlj?SoT`B_}xIi_WmoZ7;OmUjl}X2q@!6( zXu0Y%Zfc`ae@>*e0_g2}4f+`Ey$3hq2~9f7TM0034x-TUF!7`6%Q*4L ziOYV@qnXf*#Ly!t3XsAnniRuGihDP?+H<5L)PYRelzs8fYer}Gf2H+(ED9Uz2x;ln9DcP9_51(c&d&Go$@5!M6*}_-TIaa7z0P2+$NfEfK5QyY z0u+{BY`;!vE`6QSOole|S-lNrTkihby8fRdQOWxzhR@b9uWmU_WAEDS?M&H6-UFam zMi~7UL!_J6-AUk;FTtBQ1!CBI8&*S5R!>)-NSl2#Il4I3_*5{0XO>P4f zJvM6rcYDhuo)%_NZ>iJ>)P0Z8Z8G(nt2JlVNz-ywV*Yry=@m_IrnQh`(3YkfP(FW# z!pqI_jH~EXoXp4*z@VSIIh65UygA4-&^Id zTPeq3=T6?VE07I)LT<#wXP&&M7H)EW@$3c+=HF%6i=t&tnZ1uUrX)si1~N^!C?e`o zEKL)i5S~ZkhMBtKof64_5kyvU_Q8EgkC4v6OdKBe%k71B!FEFqAK?bW#mQ<{T&LYL z!OG6_Uoe-g0lPhH9V0U?HiYiD7cHfuoLa!1SJiHf*l%8>Qo8^-C)`YP%8*8oO{7Qj ztsuj{ruJt&{+ZgZe@*Rwk5T&9)V^5zYt#01gD{tsv&PofA!#nZ0tJA?{Sy!qd~@=dheBcl4w|L&FlH2BLacdskJ zjf#Iac;IRo3okB8Vg@AK1Id-bcL(y3&r@#GaU0> zsY>NM9oa7>;-T9-80dniuDqM@MSwL6s-y7vN7w4|QwjlpCDERBB?ah%t<$$P?hnV5 zCTgQ%GDqnHPCJOvIuVIN!A`b1KQ)Jt-PN@2|Mtw8%Vwl!rQUzYk2WdU(!{MIy2dFq zc`O|(A?LFC^0nOS4~@YAj$Zw*N&z!2`=f~^*Zb}J;Q+in6%j*IX0n2xHyogDw9Fnw zWQsi zu+NodDrv%msRR_a$d!}Krm~DvWAj?+0DYHLV!%Vck=9O4B3dGe=LsI-EYl1Y9q-<$(8 z=5DNYC>jHvL`L+YcP=$Ck+MC_tx#k4KMk+=4y&!)VJD&yQ0d!=x}_5IJ#tHB&=9hh z8WZmq3cMV}eKtHmF+g;XpS)ri)JXx@RQqKGm zPciQOTMA_<)1TF9Y<<%!w9JeyG)z@KJh>PYn50;+nC{}@>)lF@ed7ALMQ1p&_+Er1 zpQ=-wy%}I7plHuIX*GFcpv+hOaL>U$px;Wm0&2XzD}7hIC~8p zss6NDse|v1_xcP|i{6~t!jpr1+n{PFNUXEbr{a$eZ|TBzk21~3Ne%%hT13Zgd^(ze^BudU~Bl5X`ArXC;!Mz$0ka9Mzy z0)bRsq%` zQbu<0R@{VKx%4bKt^^x1Su}4dtWNf$fRq}dEd>>Qn0VpO7~#CBUC2lo3gttWEDK;C zEAfEcwH2ykK=xGHZ?f*QxFYMOH1BfN^ymm#`FDQ;C6Go;DKoA zaElysumusKLkuPYH9{c->U)lgTfom?a?eagURH}7!Z6)=E5HwCEe@9qDotZ4du z9lP>Em|KWMRnc=rflNv+?8gAd!7V6upe~k{eWuswf;Nc)9+HK6-3y%ullh9isKHK) z)oXS`4IVlb2C~bTBvklf7cyuA9lok`vRC**Ovnk9u0NP~OuwA~<16cDFEiE%zwaNW zX7grp9)x78)?Oa{IAk)l0|GJL2jpk&Hw~TpRywS|cx!!o_|$%Ly*32z&II=fzZMD! z3dzwC-iMm{L;{v}W3zpKvoe~$y69t2WKd)i4{vJXX=Ea=T-@VJRgcO&*fkxiZV^$Y1^(yI!jcIj6P8gz&sG`t)v4HYpN&^ zy@w8-nNDS4L-tJuglG8>*7n)F0cYwm6I3L) z)qMus3gONY4>~eHIueAM(HBN2AyV=z0v#!2K%OguS3zy_Fm1NxA0b!tSNJ+t0RXuT z!V~HYcBNeove{R)`H^}9P;#BC;I?-#PM%9@QEiv|2$>#GC(QOpMxO2q&(D_Km!Y|z z#Pjy<_;#|Wl!sYIp@E6@@#sn|WgvP$m8k)$tb|zU9#Cah4Y^mY!n=Sf8-hp%RM{1v z%1Ve8dLUAu7!7F|$9M4H%m7t304SI*fP!HM6pR__0HDg^?seh-RhIi+*cH6z3}FSR zviBws8Bkn?hG4H}ByEU5K$Tqps%*ota~=ExA`nnzf7bB-0IKZoJ&BNafGRs{0)bFL ze$OC1=O!+r3klKAjWS65??6(=3rOlXb#4Gjov!(iE_W>uE6?k;GXyg5R`b^o*l=$< zYU`cc2qG~zE`fF1WsqLLx(i_4hI8jOu&#eTr2i4zkMtJeGZg7KcIURWs?$#%kPB!> z5pDp-avMB28wws&hhgvFY3y?b`Wwj`|;v(sVZx3kP)ePD2RBSqe~jd&d+W z@b$0C+y466T)TwnfsgY3UuSr2CI`qrpv>Uyd`iOq6zyzed;3SF)WUj}xWJ(+O{Ikj z+bS9{IE@iok}ewnl+5^I{;(|M;s-*9u7{mpG#{h0QH#Xu0EM^V8DCA*Uje1BkgQSZ zQYrsErNe_>6|hVxEm=e)HFt<7pH>i+lrnjYi>fM5^ULmb4B@c+(pbje6VCBcxah~? z=)N*WlUe0&KY1#`V{CGB7cr52lxJEnyYfc9L4ERbOvSe-lgh>o+MnCppjx{vqie6eKL3VWc{F0mOE(E^HdwHhR_G=#D5cIc0S zecsgDtZK^QsZR;p&hgb+;iZvq(Ja+!??)E!;g}Bfonb^7c4#O^Lsygt!Y@=IEHqYT z9p-&Qz)&T0vLrxaj##4?cibE#jvlWwij(os3J%&H@UmgOZJgC@JRxxLB>VGEC3nO4 z3TXRSm6?od9#$HM{4(-nnVr4mvpq`!U zMzA7{T8w*C|GS)P#K8EU&zu9D<^n&qDqcK~LmC?Wsdzl_WD3+1lNngaI!9eYm*m0i75)0~VZHC& zll`xKHTZEd4_@N+Mp`wwkmF9;6}K(4^xr&l<8N&H{nY#jZ6DmXlAK*2q|`F8~; zANjX}n_Bw+py1s5&}2g+Z=?v*Mp0`%Y~LdP56Dx`=nWv_nj-&_aaaGNjJq`8XD`?< zY7#)T({;$1yvDwlc_I#>X{t`*rp$HriI>~Lefp76j(jc-mtyDBV>A)Y)mc0{~9G5c0uMEFZJn?oB zPBhQAg5I7KtJh`-(B~ng?jiNJ(z17k$8?bLIs}j>UI2N5+b_Sj690l1QCR77BW}jA zweuN#wY$B-C~rZ}(NBZpA<&ygY4*Uc@D2c>e$uJCbp#|opqS+rh^OK)W34pc1Zc%` zZ>rcLl6^Y~AH$fN`nbGtW*_kRfpvZ=RD65w0)AZuD?`;qf@BL8hnP=p-s9fZZcVP4 zs0I-gPdTPnLpief>FxU6_2qrbdmbiN^NmDXKd%)bKGF8=>o*(+q+cB#cfB!;vbh~m zV4FXGA`x&yqU=aoIgAK~ zl6ihoNwQjJAach|8I|NnBykend@K0u`A(X5Am88q8I|;y@2>Oy!6gzVAeHCYCXDsG z+^#hmqkacBxn@q?teweNaQyerv?E4;?K(1j8qEIa#53+C!b~-5O_ffX8^sABc1qNt`>|M{C^jxl{->F%HAO29FL5pjvwSzhBp2X5xb3sB!*hf|ZH zQ3XG{mRDU7$qqVn*?4s!aA?>C&rdoT(!Oq1_S`-gc;V#eKP1XnZNq7w*(RtI)7_3J z|E`uyYuyUNs-Djkn_(JsE(0#$(%T*nNVt8_GSgZE{X)Sn&uPdne*{Sq@LR+TG=QoK z8X6W!IT`pR00ORZVK6Qx&mfF*`N5sPX(T!O_;A$m2uHm9HBQ_!U#8*vlY)MdbyT4a zwmOTSEST9$b1i6!jTmLmO|sQ!p)Q(0V|i_7Hwwxld--M_#*x=pV*n3)3L(}=tXiOIhgIns5^hx za2hajClITtt!o;*1MK#7d^7 zxX2SaouUUr?>zQCz91Ab$>c)j3iC2ILhMa`)5;L^%Xb?)s)=e2Hc{e?nfZEaS_9n& zRgOw={qVzeOBd7SHu`$D>0{oV0p6!($3+N2Djhv*#i(~3zUU>9m3EYeLT6spQzx8N zUI$K$=;sBa3RF&cPXiUa4MXq!u0Pe9FK-1fNBFQt_%?an@v5tKc-?70Ev9&`NU7x{ zn|hx2oyCrZov%%VYA>Jn9UuIlzIM)&=+1htonEoX2jG(oMXt=aj*Q;yH|O^tfs*ol z^;tYk> z<`pzIzM@V$a$K$gKF~4nhGByyZy%qa@=YJ=u)K0P=@gAOir!Bd%pkag8O%!z=qCV< zketT?xAJrp(ahlf$1 ztwQ|tuzm`FDA3VFeKd6v^9Y69){<^k4LF>0#F_@OK#v!BW2wcUU*dfnpkHBcX<$ynflGR*y#ceAi5hJt+)VIaEll1bzo!ixP#o7Nx+A-?fuV*2BUn2KFH zt8W@+)-wh$4N*zQz9UsL3hr~x>U)K)hO7oAh_{RJrjleYli0(g;q`D&!(W5v9-`xr z>(loz=jDUXfTxmO6|FMoars(hnuAtAA=9WMc6-!F9{1P)n?DMV`$28~Y!38&3F#Vd z*}bw;HP5(Pts$WS&v3(`6#IP~P+t%bpg08iZ^}Utph)|*q7Mz&d*V8`=&#`QNxFx; z-~hyMlpvzC@yrmSs*_KuA#`lsNS~S@i2QOm9$*fJ{dkj1ET_o*iqH+^u*O=4nC{Ug z5||3=#NiKyR}7$Q!fYLm1}eyAdY7G~$GB$5!n}YnpD+f5Os%c$t#W4aZZMD#^c1^V zfIvY)QYL5*!^ZvLGOr%@P!*%5Q~K9{P2W%E@x*TgeC-0rZ9U<20Z8;+ z<=x?qFqf(YK3KBs_BIdU*Bk(0K-eP&;0uN#D8Dv|IV^vL{iHf12U#;LEzYP4=#9Gc zBi2o-G~|~d5kda%<)9^lz|YTMEPbl;w9y%5CeAm}Z*f?%(f9CU1jn7o-Q0fefa}pj zPw4%7(ol^eM}7Wl2~z&+fy7vWYm8N_DiUz!YAU~dpf3qi z{N9jbi?e0FKcm2d9@Vlyf^=6?GedRj2Qcu9zzhQDA$O%(7;?Kji~tWj)(Jd-m+bB@ zxn`w}0{S&@cYxS7vddRw^1MF`3=V(!nr^ecye)$!q{FR^~(ye?@C3;MMK$0^-` z9S36DviWpqENT>6P`|!ZQ3wYG*3S=zcaDG;Bmf`42UZf`6GhK$w~47rvS_X=dh6ENbLz=+h{E+XjXUJgoMo#?iL_@pmg z0E*BgITFgbiccXra4}A4e_{V{xK(^uRne?;E!Vmy@&ISqFu5|Jb_$X?U%*{IB?uHHEcc90p=8e-H0 z!dxa9tpKg@b5(n`Zk;09iFy#@0ObDjxbVTwnqWOvA@Mz~1Qp zW>>D0Rm1DC{n>-H9xg)j0M zBQ4l_w`D%8_NQmib_ymQzX|&fvlB@U>h%l^`{HiYVX8O2gg1;@v;0wuPeLF3ocdhA zK>Y>Q%K_b4z?#dv-3{%q58{$y{j=J1hBG)ZNS+)8HY+kQK0Qo_4&A!CnVg^Hsw`iI z*nL2IrQkY7_LysW-3fk(q^{<2M-(x!U(f1_sx5XaPH(jD$4cr_cVZg150_BNx?UV~7{GJYS=-++?EM^V;4=7?y9_ z??Q90=2rE^WODxP^Fvu&=cx^1(s)w%UGu2v^IN?b`}5s?PLA`e%&hF#Y8t~1Zr||n zv&TiwN!7K*LPc(+3R<3UR z=#|g;lO2JdM=A(g?Uqty*A36R2|+`}4cz?XMckQw%LB!f{^*xkv$Bu;{V*nv3z(+-pfv;!I9Og{W~ ziL*iLL*k6Fs?|sPWWpWwPl@xT;BPxn3{5=ChaE^7$i{L9THmL`80nCt{t#Mf;SNhn zCejh)8JjaYUb%toI`dm!GR*)AL}phl^4O-k^HhI8=Ls8@LJiE+lGsDC=#agp7P7cG zL?gC|{b^p{5(L`h#Y)wnF;H_~yhGgI13H^vFoTPGTe9d(IfK)xd)JfIl_|Q`KK#)x zcYhKusCZt-7IAig<|=8s1yy!=VFU@OO3F;p+6txbl7`+b)U~>5qkN(prCaSZa1hd4Mc+;fq{V-y90K;~W`_f!W&a-rp842fhR4MAmL z6oQzeuW4m7BYgQu4MewEW46$AW3y%P=aNqc+6BH5@f^uVIl4t}GB8=v!L5jK4vNNU z65i<%CK5Z`rQ2dqp7;gzyi>Cyh|K1U%CFVYVdyzOLnHd{}aal$4p{vt`NE$)quhlP1DxO$@g1=UE> z>bOg)RK9T0x^9)=SItj7JCtZi?HNO9l^!XotGhqWGm5KJf+{-C2S5H?J0@NKSpj@L3k#w*{)HYRIx10GNFL8$p{rtFm%hBVRXI=Qcz z9Bi|SeCw0umzHN?*3{lQ4!7}>W!r6uUK0`{6LL=q+GSb}N%3^$+lm=p(;5SG9!NWC z7u%C;6`CnL3z$ZlPT8q1F9oB*wdTBsJ_Uul!V57k4T4M#a~_c$P?a5rKXoO*`M1B_ zNF8`~{IFSGBu*BUsXe)112z+M&x0)!t?6)}o;rwUk2|;pJ5mm|bUW^A?W%bZMtC$S zh?njP_G8<`N?q6Ktk~P`EBn9wR=3+)_V0^uD;%rTW4c^At}l(+Vbvj{HwlWxLT;k% z!nQX=c}f#@Z{J3Jb*fCremd04iCTK+mAlLCJIH*f!V78xe}&L!@V+-f426~5=Qwz~ zYBM{KeSHF;cFg#&;OnW2%q3{;d+7D-@gm;(erQlpx}lB{KGCM?En3pvbEK@wNaE4y zD)Mx5u#&%dlm%LJ7q5Px|0$k*zx;V5!FnUL9f9GA>%T<2ft$)h6wkxxiT2hk*%sP+ z2IqoU~LS1+%GYs6>@G@b_b2<%3g(_0-?n&EuQ z#&`O11G=BoL_~@!qb2l}e*e$?Vfe)&A9Fuc#;D17my-rPW7 zzis%xssGh$+ooer%PX^#KjH~#NHVzOKK~Dq~8e79TG2xHsx^X?&b3X;cZ6jk(Nb z$L)1ChUW>wIcZ3s@LrxPyJRcWNy!&a&byiXiSe48Cegc53f@hAyrmiY-1*%aPu%7T zz%;2V58}K5%hJrxy!k)%o>ryuLEYh;wIw^km3#KC?3Y>V{Vf@psy@y=1z6)pnOp0l zXNPGL6HwmVe+(VG7}>Eiv$Ik=t}h?@PmC372FV)|Wi)}K%qgSfd|K5WzTuAH9Y?OO zU0xshe_a;A;s^ovg2ydT?w1(ufO15jW3aRhX>nG~URByNup$P9p1Q^3ywUz~Cd*x2 zll7ogFRLb~T%D3dno3i1L*z0ehL1zysu;ijT%D;Hk;&R)b4p$x(-=;uwt^{j0{bSA zDJ>Gq(oE&Zl3Z)?JdJ{S_HQwd+O^yIaL!*kpJwIf=HR!XeG%ku({7zPwh7`nu`1EJWCs zqli|PAt_@aTb1mGPKx?zdcWiXak5xs%6hOm!QT0u-L4A4X5@+7XK5WpDE)SDzr1{a zAHzE~01^;24jLtsbq(6PNYX3rbpS8~^ovuS-DTr^5pCqgH#A1*g-vcc;ZO$|-jC^t zvBV85yF$cR1x{8oo-5$7&fJh&3&+kW)+&xQ@|X45VE1}pk`VJGOG%8V6Ow*?MpK5o zQp|^aoFS>ciJY-N+!w!!aha0p)|XVnL!0=K=--!Eiy|&v_QQVw;!1HURWT*t7l#RU zm^3AIz;I%xQ(Cr?DrgF&JH+?2->O4%A(G8z!zCH`#<#LBLC9P4w9* z(Q7$vvV^Rc>sl!|D%GKAN|NwH(`QVn@(vO;WAhWPYJ^j$VR|TpR^SpnX0r%oChqZ( zBy>XtlzL%Py<~q3>{DtuM`fc~zpbK-WuqMZ^psSE=qu?*ukn)9ia1mnIm!rOjLJ#m zo|lWTF}q#lqeM^K(S~r>yvR^3g$y{@Rq3{q)PX>o4F5q=EvmxrpopI%fEB`|#H{Hp zQvBI(Q8NCMz=>kMn9<3V5J>kT42%9rDh55PzE$@k>=F6kqTJ{ARF_ZuF4``5#tUO9 z5bSMX@OCahVc4pBX@xJ-+1};Myl$H#GU++JfaCKSY@XkGa21jyo7hG0;wK>)biJ@1 zqZ4NDni;<=$P;G$5)dvKkR^;2bJqAvic`W=%kA3Z{q#_h6vzx2Wb8$&-}61DV5}LZ z1i&uY=i4?A2Rh|t-65S}wrz#fwr;8*fa5_Kh~uw%zZMWjuzgi|+hsOTephoQK&#p| zF-WtP1az~O3cwX8cAwGat>4(q53IG6j9(=^_&cUlud8hHGvJ7NVJ8sM%};hRS?Noh z8XrIIC$c}V^}d=}dC=8;SOBHIj@i~)!*UN>ekOJ_N&HWOHFx_USc{+eqkBr~KNtp~ zuS6ts|3cUm`?5kH!k!4AKv}NxkBD$-xJ4;OMs9f!G96znqA9-+e|eFmB01v}mJVn1 z<7)XJcr_8hpa+_!OD3F@;T+{;Z_!TytUlV!|2qZ zE#@T=F7n8^iZ%KV{V~$Hb5_5GXK&#&zL3fs&=q<^H&?7*!_qgSOy)^?HPzZIU|(cU z*!^5QUss+v52Yb`t@3r+#zl7_s>4Am3Rs<)`@)di8!*)lkr$T4*6=0q^0!4wAdaN7 zybi}F)#^;x4FpX}v3jb?`U_(lkqnbl8fAym_#zWrJXfr-4(q*SRH38di+?x~uc92w zw%n@3hD*u@y4e+$2WDX_^izy(lr1DH+oS$rL@egu1@3#&h>XAaJ8Z*=Do^*HsbEG$ zg*d-PNN&H4l)Ro@sbMmXbm_SNvHT!udO|YQ-(4lDLbC#0g=*uWz9gHNc%6*Ajl)skIJDIB>J5i%nzwX< zo#?)mnx@pN6*F3D$6=BnYgJF`1Dp)OVp{!K!`I=$(Db5Oz4qBjQA*_asS14}@6_-I zFc(bg@jrFv^`&8n^IZ+wsZ)ZS?ChIZxoqcHx!7m&N%?z9$S)TV`;)D)J_}dBSPFv- zrytKYKN$oW34BGP#rUJO2k-m&l^wnJIy4L6SJNpB`phQh1q_=GaBr0Lj5DXBqvPmp zuZzxl+~D{lA|BLpz(q_THXf>8;n=)n+9@N8;!}JR0{d^o{ZZ#5~8VW;B7c8-UtK+QVwYi`8hs={h1ueT4b_PX+0#$4=M zm@N)K>B6HLPAz>6DZV*1qn?>)NOXhEd(d%;qOY6Elgpe8(rb_+e z&!fZkeL&6Gy5RrB)fHjZ=c9k8g2e$m-iVAiCSZZ-(jhA@|4nqp`w*SK9R5pm_748r zhG_YpHbnmzzQj>2?$dv>A;K7x#C6hZ%IxJVUhiIzNr_Fn4H|`;)~UX<+I?({dYY

|JD$AO%NR4iDY8tH z<{uemkJ34+0Mt1CBcY;IQXMD&23p%p@g*B z6{%LRFcc^s4@r1$RM##P&voU(#m_ zp}*8{j|(ZOFDu9Z&3k1=C*Y3*Lv5HXe${*Dt=tLME~8WGZ+!Qv+ilbgJGKi^R_w?= z;{)Hs7qO?>uGGFjg{Qy+%|0Z{pm*$+rl<=-auicE*HdWHxsM?ehvY`zAm;=)@J87=iO)9a%)e6#|xYa@vGBXquO(u4g#3MmEW*p5=12IYS zW7|p#P9SQE-E*-x3?V-V>d;oa+g>`2m`?YuSh~|Jen4!(cVqk2C4zQiFEiLR=C>Vz zWSLL_IJK&-!T)?kwM7Jc0%&UAVZ;IWN0XCQ#|hYR=+0q%?$n;N(Hw29C7SciWprtE zxTkU&MunFB6Ww^S+;o6gwJ;kV$(hxxu0&1?BrFG+tcN^Vav$KOPsL0)0q`yx5Uoss zFP8yd)^sLUENP_55)UW%nd21}Edi1KixxHfxZnN$v&yMhMjrkNWWl06W_q4-m`2WH z^=$415*$b`?-Scr6^>`hj^hVi0ow#ie8^VAsu5=SuPzFlNDK4uZRps(RKMp6ik$mf)rFo71$L&=E%^W0ZfWp zeuIh{iMub!gRT(&1w<@I(U~mk;yM4<`5_;byMrIb0vPNIL23p)NC;IqAh-J&^$?pq zx7WEC*{2AIKJRIE+=RK{O)(%1Q(h_9NQs;C7rFC{bzI!)P9jJ(IT0s^i*l9;EAY?m zmd0J_m=R!5RXJb{kV|Vj-Jz>-kXFq;th4F6uCbp8+$;fh+fN({p0ssbnGbyE2!1XJ zYa`Vv9%xM(S#>)p3p;g8GbgvQa+N#wT?^-Wv9WTx2(o(FS?Q+a35r=oWpTfQAm8_{ zh9~ELtm|zTYE?h!ywl|{*p7Q|?<3e>K^@+aWv%b#lm)I{PY;nul26s|V82y6kThez zEol65luL>+M*?dK)=mDMrYj_wMrRjZ;3eI>8&)<4DixDz>Dp9LS1GoKU+L}h z${orl$3K3GJ!(#Km0`jm@5nXcs0sICnxeXQi`VT0N4zXuJhQykWXuPp-DIg<($HX= zQto+T-O}x4HkvADX|TbV1hv{^xmwatUQMFwO!KqkgTj8GDN&m;!@zZ^MKDTTd1Ey9 zf1B~f_-;kJ^6dl23{v}mKo-znDnZHBIkhJ@{Q(Xca->C6<3DQo%cQ{L2rFDwVk(p~ zZAnQfcY+e-Q6J)e|GBCu6vjc2Fc*~NpzXxfWfO{boB$RWk5Fn8D~#0&$z7yH74g4F zNvo14P5j%LtED7$(l<z`ie zC8q72B|1|fIAv1RlB+0ur4^kH?4HbiCRSMJrqv6Yd(CCm3TCHq*lTpAiHuNz=@53( z2>lorO8BiZB9?%*c9Zz4%tS(}scZBT3dO_Ok zMAeK0a>~b5Zi#7`KwdRDMXCxIm4vxM^m5Y5v;;K?-2_E72}34wS}D^6#bnY3i3rnv zWcY8A8t&|o&JPh$x%w6IaaM!s3X(VKmIG2Xs(uZi73lhvVc|WRd4y|hmJsaK{w+je zpY|i#Tk;J}!E4mf;`|3dzhXEZ$p~51s9Vh%#(6GRv?)7=J0|+DTXsk_Tw-41FWAi~ zU}?GhF3m!es&d$Jhd{(+#lks$TO^6wNru*U8;t*xA{$ALIm^T3-&skpC z2_h-hR7;&}v9_$t17a6VGD=13dr>JvbvCi`Lpp`Q5M$K_RE;gqVz9V>s1WzMzgamk z+M~_J22?KPw7r?~5Knt02;)xU=P!xjD!eN55gdJCd&R4x6lQZ#DkBpGlx}osoU$P5 z)LP)9TI_7SfaH|4r9nUbYEaQn0+6uJ2PO@gT2Dg{6SD|HKKvUkGZEJrOdd2hQF+zc z7{E`drRtz&olpmTV~Hz6#YV9bCz`t&i?4kR7hb-PjK$r{)U`mj)-dZbUcJ41gS7EN zH5=_RQ7zcTdj0~%G4%u4QL)@@Duo<(f$;F_ng#A3XGIM}^OA<{NFO2Ho8^{S;s^Mp zdG=``)(M@f{A!Tbna1wW&Q`=%Asc6~g-4=VsFn#!coa%&>tBzs#JZ-X;v@6TeV&_L z9g?nYSQ8--7lCQ~GIh()Z(>U@P9SCaEahjR=jPHRx9FcJmGByV$p#uIC?OWpG0J-8(PjcFVXS2IEfwW$qvc?=wV}vLf^ei|sW#Anv&7Jg zSs2hgXN6Syu2h7OZAvh=#i4aoYZ9+E^q;>fFKgBbun;8uTKqI`M@hE$soUlY+~Oys zk8g4*j_(yf`arM{P%P+i(H_eEPWdIE8^KgSRs>)e77-*%*?2OYt)V_xQseP2mNeM| zvgB3(ZdITGa8>2=M2$abLqg@jf91*iJnR^Iqd3c_`$^w5+(-!wl(CMo3mm{k?ziQ* zfinrYiVk4Uj)pKv$P8h#I)W?0ptj3MGjfa;kN~DQgVKDYI1ivHN_;avUaTicU1;7( zjC85Jz$#KD2HHYR=Sf|tN*{o(&@5%~zoQYw=}g?phgiaaa;Qqgha5Wa4>^?nbIGd~ zw@|u3zrWq6gEM3=UYgApjS@X(gLo>%KqqTCI`M({F5df|BWU=m)2AyF)o3F&(j;{89KO1SJ7pK4NGpOQli<# z=8VIyas~aYd{7xv-Xf?7t<`a@=Y`9>C&p_l)if&PjtO4qMfpVhMad|AKzVLUR-8HxYOnJzk3rNk$b;dS{27zri_j4{Ha&~npUZs zGO-ldg1fMTWjtov<8-Nj>a8P>B(Dq&*2lO>(~mv~!|sJx@@em?Q<={K#Xf2H3{$i)-qca?~d0rRKBrTr6SdihZN zq7dH6?Wu=Zk;U&NcAlP|uyV4pH)nQu$zD@BeD0e)EwVKZz*f0VW9n3ROG>!ohzI|b z?pWa-Y-B16W6MEX(7Tr?!+E)c@;}5-=k{qw)=P6;Ky1;RIo6>+d<0Veui&X$?K+Rg z{T~C*mxuPQ7yn(-9cur#d1p+<-*3M5#HTeYFy3Upf^Oekxc}44XwaQ@F=*eb?|%Bh-GK7lGL5&&5~9G?M(&>XPP9q zyVvjA&Qeyd?_hf%J<`kg!awpp(Kfp7REg1%|&#a(6z1+FAA1Ae*7LUP(L)n3K);X z2%lf~Q&_7_QS}O-50y;R!eiW)6KYwBnXW)*R*YlES9Xq)mje zc21XPLUDAQpbtf-c-__Y&VG6@q}|nuM`XR#t23qpr+V$r@}y?I{`vDFD0XNq{4Eej zO^8TMekfmP`B$Yu!ZMhPLYfimwSAhrkRIhTwdz;^Z2*_!fVHOhfj8f~w~iKT+@;UI zUYm3wk~7=~Cm2W-<4@Hamdg9n#?v-z_Kn%HIanr#U!1<6#M6_3AV5FW#$+$N(F9;B zpt!}+ti-QXq)ULZoJEWJwtfaYV1jHO@dBE0EC9_cyOa2v0lxMLn?H7~g}^uAx-o%4 zzfhah)RKsuw6F4p9PnhkZ#r3-WeA`UrQq_12=Zlra9)CZ4?w*KQ3KvEn6#nKdPoKN zHdL<2qPR*u{!2}(T$P;(@zX>uOFSyHl&H7=)b!Q!{WOT!IDAe#50fDAJ;I)5hSYk>z@E2wey|4iZdQ z$GC7{szOH_tpXn z-6HY60{OBI$zM^8>$faeJjj3g3DAHd39tan!V&-)DoXeHPbHT&!k+cRtUF)g4l~{#X(&l?e4e|F@vT}_W6Jx1)Jni#RAi{c@4FNQ2qBmttyhM~ zkjva>hc?i-z9BI+Wm$KUX`g@RonT&Aha(D^MImc8%?zBg6m}hWp;?S^Mt=zcM8=Oy#SJ$ zl6ZO1{UY_%y1No~Q~Mr-eL)_Ky_yipt`lrid-(PC(p_rCgxIOL|5q0hG8x{_=K+E} z8O6#-Ov8*%;p9?8rRv2bKAQ#-OU5F1IQXB6R6GAXb^*|+{W?e%EHs*hb!@a7NGUEg z3g_y=JHd@4lP>6x;XbKqHIiaA{8`%+7}G+J?-YKfe+vt1R)|-#%Ne2!AE_4>`fguw zm{{#9^c^N#twljqxDgwZvtF>d`DmZoQPmiy9MUvgXD}Sngx<$wdDOFm(ETYxLDQH& z3l0J_1yk4^OXQwIt#ZH8eH18uF4mptEI1sv>_)bf zUbo)H)u;;)(nb_fJoy-ebjhKVC_--K&LBUO^W-_;MCX5p4rP)^wc=VmA7evKfU3QQFtfaUAovQg zl9Ii58wJP`;mZW(vKwzAdChl{5^0}Mikv?}`9z)xFX_VBcYORH1P1js1}z({(~|Sd zCwgFC^<-B{PL-~e<+2*cvKm+o+;)*&NqJ7UL$w`VYJ{;I7ZJ;{sOd~tEw}uNlRixq zt+nQk;pH+-3_4xGK+)ug?7=H#ih5#S7Zsdjm}~n}a{|{1n9E=EmSV!SJ z6gYoM6aS)i;z?bJSSE%0b=h%xg?3kX1IdKe0OdA8F^Lr$VRrueRqqc-k#Z{u82iqK zEBNmQpWTCpWeeZJ1NlE$HpV7K8*zr1u`4fv-59^6{cOU z)zc@U*A)1(@@rEW&FlCZ0RuJ41RTN%kGR=t&^P@B;lE`{^zcM42p(in5|y?1bQU_| z1n5jW$GVVz+5$e7EVEY;d^hK4=~1T_95!cGmHB$T{AA6_Z7x;S15m z`QjB;+iu&1Q(fD2-cKq%?xiEOq|{}kG;rJtV0CuRrm}b0O2kF;(PaC27K}x(oXWe!-o>y26HRHFMFn8 z-DqS6gG0AULL=1&J@pWlR<94}xr`_6OdUYV@s+)SF?&OsrAN^;9|K~Jc>M}+Ec^;? zfd_L;K$}gVeAzo=9t!XlHHhkOAJ;+26d#J8xR9s>?^QAG+zS+gau2Sw@_yvtWj?H) z!JjUs*De&MJyW7y4%~L%i$(Fd-K2s5X}}{-s5oox6y&Rl!a0%p$N9XWQ}&9y>N<&; zpt|MvElQ%t8|u3A-LFu!2A`Ajy; zNXw~8fwuD}T@sO(mgrJV*lXJ^gU?q1jm!g`(6zQqZr+`r)a*`r!a6@MRFGAZoV@B7kMwyzqqYPn&r(GfLS^aoGg79M#3mFUu84SIIRk(!V$Dd2yhp5#& z6p(medB*`#CkI2C>aCl<1QmsN3^r?U6=M^iq=TaiPf1WLKD{;eDTkVI96w#=$@k!! zDM%hGf9avtyIlzGRO#6Y4m0exp=-o1hphaH-g|%)3`Uk=8a8NW^xJp>uApP zhS2>rOS~mR5zg12lIk&xyu{d?zWX|}QY{MM#`7@3>1rt;{w~8h;}=*RtxEw4-|@m@ z3I9r(47=$Aj$8Q*OtL8j5nD(Ai&+Q3trUKLl5Z_KtVQNj+TPI%gHDK9eBF+{A}gC8 z*v^;FBfK=0fJGlo>__DO3QS21)9Wk9n~lKYdB^0}fq+ce)ue$-Db+Ig;Q~P_5LxfR zI-Q~~T^9dNYM9q!G3o+QfMNETJIwuu;9?5Jmq7w#WH!kM*fUfPQiU`eGib(`q^zr3 z8)!ztODeXJh@rb3!BACbmOc%}wZ&^6iEpKbNDGD${B}*qVidg%iv~J75_s?O`)+Yk z>+USc{+ZJb8Af@vV7eKbE6VOPJ)A#- zl%lZSm@K^xLDl1eWd=zmRP>x$8{CoD^Ut8O&CN3)f?2fO%Cx;%? ze;KJyakdt^x?wA*AM)eG98s@Tw&V2fgLgkllHCy}Z-QU>O?H1X+*mEe^SgNYX*av5 zWIhkE;D<~@3izn)s1Y{t-S$(fUgWI|*F2ld#n|fY!EXwKcONY~YPn7Ng*P)TCm`W_BL30nnyK3Xgis8bsF`qWB zo%cVhEq3;FE|emS)>iEJ=)Ib2%x~uaZ`?k6x@;Ud$HvBbtQGM5SgDI`^&IlJSU*2K z70K&2oNRT~8O>UWqe}+OTBwW-8kmX!p5UO<_ZGlX;Ef@P5gIBhrd#b4CTYeDEm)X6 zQ=->+Wy|MVl}iS9*yvWiBXV>Y2e3o2BabqD$xvW~GLqRcmf6Dloxw!;6gr|wlYre& zo9NqL8?(l``w$*cpW+BWoq0mr<5)&jycf7Jq^UqsT)E-;Fq;TtCH{KRGp;^S*x@6Ci2^Y6sc zXPQv0D>@Vv_w@zLKcsQTX&>NMb>m}zYNXiw)MOwIsV_N>JTa_nZ0!rNH#@S4yA zIIUNppQcW&y-80qd8^2Ws~jK$7&$}-UGpAw%yM7RY`+L`MW5R5y2Ik;szHgD9p?FW z5>gWF=k8=bQQvk6?=WY7 zs-(q_O&5^k#Q6qnN@QZr+FW~L)s3?;nXhaA#73ZmBhoO)#72O4B+fTniqbBmlok$# zDDk%u0~Q4#dLp?WMZwPqQkU23RXqj25m=radoqZ7=80vJUHvlV=z`Y8X^xn$hV()g zKU7#=Y<`ksO9ab71jw=Cc`+i-zK)$UkVGVvXa0#UP~+w_IPZ0QC7SxSl;?*jK(6#i z%3xuA_agc=<((^9*nCqk&@O$?F8u&SRB0!2_^|PXUHnrCw|h-=MBP$TT)5$8f^Z$X zKqB5NRyl+!#_+adO8cvY`~oq0-q-hQi&U)_A1-#_volOKRdDH976~smd{cq`W=MT7 z+TN{ym-`P{@uC>wbd?zVPXz-R&IBFv7;cIZ!>zrvkpPFxFPh{;V&8xM9{iHu8KryKrN4t9TdIJu2C94dse}AH ziUjm)O2ULW#~G_lgi?I{V}G(dc_s?FB4+;!sgoohCv;bY8PfV*HA8jq{m)YL`!vts z`)fgL*Xs*ecjqL0CeJ)P1?pUSBjao0*QIh(6}6Rvakpj4tA`-YMIi*taf|VTgT7H_ z>R00Z1J{NZ_~fFid4Cg1x>t~|Qe2+pW?JKt^tqE4VX7Kvzh0tLF(iZD2CCC>XMV>O z1}LHR9Zwh23j60KLNQ+Qc*#o=NPFpT&tQU(bt;Zvnr;7R^)xRDP(*9L3!fYqzd*g5 z+!0an{wWzgw?oauviB63?9rK!7kHwhio>?g|)lrKQUl$ zy%HZIEs?JY&GK;yt3KZpPIROhAK%F$#I5o02zygc-0#;LbT+3K_*~`8@$yil9I34A zuEG0{G8C!wqvFb_p6h53USB%1(ln|x)jcMf^g98oSZrH&9(b=)_ox?^B|ZH zKu{F2RQ~0ZzI(e@>ZFn-qc_zf+o87X7RA<+=NW0~E)E`zF<%tZLsjrA_to|jan9V&KSJ6fHX4s6yHX*suE3(ie&@`rt+;B8XBwg42pK*MX;l|Y1I5sWGMs}*7p1_&4l=_! zuOIwm^|coTLlxBnrHV46Z4>@>WuYL6lmXDkOBiqRgGdAatwwVW8!gOOArWa9H14ZZ z;tJZZqG(k5AwaTq(E6kl4z??gTA+xeV#YY5OM-_I(K?QTIo<({df1ZXobUvh?xuO; zMHlB*+-GsFyfniy?x;&X#HV z;^)tvjf^g2?qp2`p5(5w@b*k5#z<@lDG2sVe_~=|c+Ox1u;iMZESlsBv;6Q7Pu!k} zx?|KNA8}9JWMVJbHL!Yed~4S4X^!)L>aX)>J{~Pt?n#`#Zp6qDP>}3Gth3;W`g~-i zK%SDjT>{GbnXwEjQ)Wl~JT8N8HME<{s`Gj*FQb1fCPA$<-H)xMpIoAl+tLWkFaEj9 zjHWtte6*u5PeO{KZbgiuZs@n8C`*1SgL2ZD8IhzCu0a2AzY@(ij{Ls8K(@5){tb~# z>y4~`NL92-CbfWpZSF#`9)avWFfbsG6gnm4@IIA}W$*a{@k`ihsVAi}=#-CCJM=Bd zPG|2@!tb=m7_f5v8>A7;Y9ks%2ea|`7&5c@`{A^m;oMw?A1d;uaCbP4WOsNZNn0dp zQTNxIGVomWI2b=e>^&X-ZTAru7PV7=>Q_>*iqsSYb>y0D8~8u1>h`4Q+U>2=lu;4? ztYSNt|EglmQzHjN@)(7-hvVI-k}1Op+|(6WHR47?#DvO{c%JqYx*gHK^RjpAr3NwxRO?^>r3pXfXwH620}5Qs-8fQj)Y%|3HL|0R@B+Z8`4m# z28{#<5Dof{Mg2c>U)O>;VNUlPed4^%gfZrHcgL6uqyQu?cEoznph$1uGN zCDa$EMtL&IU)r@7;BZUN#(_hd0KTzDlpY{Zes<(}OB)A~$1bI$*M+Gl{nl==$BdyI zk(xc{hKiw6IyUA2a`zPe)8YZ5Dpnixermo)nt;o-|lNq;_C2LVUgR`Al}tETjvRS|Gj}# zm)d!1*TtkpD}&z;;W*tACfc9*K6PnN5&q`^xaJkcWWw5X!kS3C@~mM#0A8E z{YrHH9aQ9#HWIa#W_XZ)+ZqV8YlQ0#7$1yaWaU#ayG3d<4#tryJi|k|3G+}lZc`a(lh(fA*4x;|?RFV%XdQ}*>ul>! zXOFJ95RyNIP&mRQl;$Ls&tBN&-nBWc>#|Av?>KL#Weho7YE;D3D+l>v2 zFtN0ZE2d{czl-{RC_C$@HrKRY2X`r6+}$be?(P)V;%+VO?pmN&ad&s8xKmt;yE`Xz z@0q>lJ2Pj#^AA~BfrOR3Yvp?GpWKktDE37&+-;FCBuYkNQX^}-O`075M>I;_Eg9JI zFLlKB^9+gk=S3LYT){@tKbwf|W{ZX3g2C&l@3uB7dJ4~;S6J_IeOX9DpFat3G(fQu z%E>IrLMLjrfNaxV?#eu=6U!U5RC7b?=W6Wq9-cj4?T}Wxfdet5oA4UxSxmM zl1mLTCTIi+LqRQMUy$=woVqE(e_8cn)y?yMw!qElRr$ITjp{)R;oNQD^XuwZJvaV^ z`-N)WYK!^8ySIb3L@+A2h(MX_^~quc@q&hE?i5=0^Uc=Kmy5kyj1T9L^iWMtxG3zL zVI6hq@<&_*R{W?>vd9ilQ6a47$+5!6ankxD^Ojsb-l>TD$>t0U|RGsbse3# z&2@|GiYo!|__HeseI*Bi(rzjZJS05DaM1@CMZibmknm^e_7`mUi%VoemTbvJA_KVf zUlmO|$JFcLa=-Gav_2=7*(|pd1$ z0G1ViI}0cb5R-vB)5aZ1C=5KD*kZq8A?C(3u0QCGevkB)=(pkqBRs*Hi+9@J=UgA# z;Q81$z}+=05oo{co2=)8E6%KsJeW-qAb<#*I{D&8)QH(@j&mL9``hC(enak86oN9q zMeeWSHNaKP!tdr7W)Z@I3b02jB8bigNbv0v@Cx%4V{;o4mxq}lt^mc#JYk@7BIEIv2L<(4S3{B*X$ zwX;2-?h0>YuRC%WuRE1xRieu;ZFi=ZFQc;PaV=mA(c%>`kH_C=o?-U#6r0aGdB|B@ zFzM-+PyN{*YSME6`{gOn(KdyJW*t|5gxgkrTwij7es-S4ak9yQGCiGFvTr?4IDf0AE-~@nxvDdQC{J*uybBFnj>u+(6O+FLfGJgEmPh!(C^IOIo@8) z>>oB4d+D;|&)?N7oo?alqT_JcXbyCmHTGtE8OlvCdTb#le7#!{vu{DzbjI{67!Ek* z0%?u}b96pziG8Btq%7W!)-eq$iRUlrDrmlWiv{PkBjmX@%|gAZK|CMaRj~~=XxiBZ zHh8V*uYM(5y&fki#VP%W`hAI(GEb4wrnRu0bTWy{<|Bhx10P$1zA=L)Fzd^6+hI|t zK)k?P&{YMC6imj1S$Lu%?3aU`E+~ySv!M2pm>i}s&BA)AbEOJ$+y?8W>Q+09mFwhO z&rIo_hdC>fod74v)Z@WoenL89jMW-0G!OxPv~N# zJ0#BBBE9GxjUt@$SX<3YwU=_sB8Hl*a1N(FC~}ig%Y$sER^zjV>W_YQnyM6cHGFys zFms3Bv*>DGX?nO4x>SanEkMW4RQhDcl`W*k&J#Jos8D-zC0K*NT)J?35RJKfs3GXT z1@6i%rr_xzFkp5)0AUdgqax$6K_BZ$T}k?gXojLJ)*JIrmi%!w+?!o(xH9$(tSco8 z`_@9uw0@#)bi-f49O!f=6%+GCv6Sn{MZtN@@Obe4YVi!xENZSv#vmi+;itlbDQyQ9 za|P5E>pcF4#uI(?*H<`czdNk9=mM-~npC?o*i=2FEI@8hmchsue~nw%`m$(iTk2x$ z-fH%==7$vL%0M5w?*bQoRz!LHHd9zROpyLe;1NXn@R~W`MuQG8`tVe*ome?#$Pq+v z5Wa5@G4x;&LzrgElW7R+hOut*WSpb0qQDa)nIkV=OOSktR3I5m)E%ds*@F>|upsz2 zAuD;k1$!M$J;Rq@`OTijlSi=Onco0cnq?_+^Ta6QJnw-i=$ANFF2hIg%BseON{&XTP+hvjTIB3!EzvUz;V9;$(N?i3sVpFL%uZEeXi zx^O;rN~=YiZKt2ua}Ih?e{Z2@^yvat_!whiZ#EDGT8SMokuvW>GthLL>L|YiHPU9n zt8+!x^8Qv~)z>6#L*3UF96@z!~3(J)?uj$OZ%A`|9}Dl)bOq z13RvfsxaHz0xpXuOHaD4bSmXd18qAmH)6N->`fJIFW+t;KdP^^9hG1z9|nDYc)6wy zS7^6cg-W!nUaVUmEeEr8LU3zyZom0?dd({`{)IzL6tOdMQ(k_bS{KYV9k4o~2v|*d zg?9rx>!03?gt{OHtP601b1*}J5h4oUYWDAz8G>z6dfkfSLa=KN*{EN|CC2b%#Fw@N z#UDkg>G_#8&U*#55JMLwp*wUpmuGQ}B%`uIYfHQ3J#BpZ$sS*f|J=6KvM#%dQxq7+ zX+koXqRj3e1-D&?%v?|&80R?5C6TIpa`R)$)%(LHqI|S^+1MlBPgNLI$Lf~0mzgIb z8@Gwp;;cG?4;AS@@IFtr&aqi9 z5htA74Ol4#^O*PaE*y$d3!QOfc6)R;;_4l1d&X4}+TiJ_@hWa?ro|cyaQLF@1=(3f z=1$zHBkiQutHXH!Xu>Q4aVV$-G=VsPZOZCpOg332yi~rywA?sg|UnI zC~*FZroPxCY*ch9)}2;hrZ@=KsNzzMeglyZhX@+B_)LbNo)Qa)G8}LL`rVVB`rSoD zkf?G9*UCt{&E&YAD?Xi=%+?H`wrHI84AAB7OLPSQ=ynMNWS%?-XhdXO!p5M7A`%LK zKNzHU95k%YhL(`eMl+JW8v;}38TLT|KCp)UPU^r$ROyKj95W5+JMin4WHn~XlX1-8 zV;|QRXi~2nlir?8XtXyGu}puzm-3V1Dk-mhRU>5*)w!|iPb|#I5NI({7}-o+Wz$cV z|4=a`+}>0Ssj4{rD9+)s%28wW!-8l=)@EJ)T%7@)@A~$lF%r*HV|us6tet4O9n>wL z1*BWQ#S221Kg0`d>N6G1&Im$Or5k*BY6ts2PkwD66F*juS$@7Ms!>GEMOQtd_k{-%R)IZ-+03o;Q!iLrR{5A8J<_ru@C z{D9?p@=YtI`S9#G?<5l|tgO|@6FbK4M5gw}z~G53Qul^LR3L0^>O#0I^6Bt+YvXG3 zsts*CZHSqjzzgTT83go7r?ZW=>MYJ&OKB2PjP5ez(~*@`_J6Q-2b0zz`6e8BXJa&~ zYu0&8y>wvMa%?Wza#~+=ReE+d$WLo6V!%$>vUd}B0(tMaS4niV9Hh`cEMG@7b3SF- z;LjRh(a~ggV5`fKIEScY9{5RF$(+cX991_m0NV3nD?`1n7bIT%N!ckD(X%n0(6 zVWn2X9Au5d5w3)&IAz)U9kb7!hYHI*@-8*Uh_5A&oZSwMD^vN~P3%K^%zjhgj1*Ip22p zvLDiRN{Bap5gr)GCe7J05*r{*;A~0dZ1Ef%jm-Y%f%k3s2{X%CrSMCDmvIPOGBm0C zO>X)Wa%7SCL`s+fv0!C*KX4PbDlHn%Fg;{hDTyRS(#I3KwP>tF;9>XNV>;E4s3!_7 zz{x{KlR{SGAi2ozluP+YyH?jVhEv+OuO#J2J8&Y;OB;CNRYD@S$lL%#7mAaS8E_{V zQz<@@jP5UmwomF(QNam_{Ckn+;D0015b@K#8w)N!n`Z#A{|R&W>^kCpd1OyvFEH%_ z|8IO7{(gdRqn<3kO&g8>&a~kb_Fidi?c%CoAa<9Em7tA_^0Pf=QoL__pBAU%guVQ8 z;)7n!d5SJc&xg1NCL`_Qn2^7zHUxjFHbwuU+H^B*2w0m?^|bz_+Q>&xN49nw@2b4^ z3T8oaEV&GeeZZ6bkd|uheC{?QBRn<9v9yV7ysTIs|BT;XdOa$G%`*=cBs}aVp%!2P zAME$^PO%DCR1K)w$cj3Ng4KmIs^mkVVoYW*8HiXA8^;y4U-D>F>BB1!-bb)dH5P;x z$VI8pCgz9AFu=G@_BW$(Yk1B|EG`-LwTMZr+4&o^`+VCoyEw6 zx+yW*yf}r7;KOrmy+$CHNbDKngvc2s7Bxss<}cpDPwNElLU%^tVQA{FqY$2l5!;iR zgfJoP9fNHxqLc_#?C;2r68~GG<^3n4&+#@_%$`{7!VPj63hkB+J*x zWmJ%T#?n#%nq?;ir+}5kD6mml2tip&@9Boi^TaRyVhbgay-7?d7O7}2`;hv@`lC|?{bEYTDEdkBCERB1*5R7sn3(epbN@{SVQP&p!GA+&=1rNX&m zw{hW9`=C31XBe5W4?1wDP6Mj8oCmx<@|X>B+O2R)l(oIOjE?Y zlPCL~hvO4SegP?t-#)L;K42R7LCP=yM`ny|QVK`#GME~1&lh(V>Sv9fnjY5*bdAn} zkFU7*9e6xO+lbk7c?p(lIf-XD0a}(YAU%hC&~@cxkjNb*M$Pmg^YaJ`_C(WL1_Q-k zR6w|a3X2aOYl06RrsOvm7hIzQ5GU|G4u&nk?5#C4V8zA^&`vw!>4OXWyg7lNH>kt1 z`L;$LDVT$Z4=xC&1tkb4eH;(!Hf0DJS{8}GfctwST(QzSgv#6`A6~5H5d5%0c!Dk&=26LF>Z_RO+<}ZwD%@pv&AqH;TdmQ^)~SeJ|piUw{*jk3^}tx zNCCpcaTiryC4PKZRVC(3uuUKKW5zW+jVcyb8@zEf#9LbYNuJ;AES|b0O7C7^rSOYn z!r1d6mcpEM?#H}c5C^^uL1$0nnT@lvWQR$vLb*IcF27Sye$=m6kVd&uxeaHDUU5)97P2P{P18W zh+|u&ajpbPUwgESmz1H(2%~14ghPA#aDIZh21Jr)o zfQ}B7D&PwbI8W?y^lclj#TIH)HzNDt<(I#^I&E2!l=bOqx z)_uj98wXsVm@#uC(mf`!4&24PwQ0w-S%t;^X6`|A%ECGN;`}#>1mm^#%k%2w`L~sZ z&nsJ7xw^NX#t&{*iY#=_2t3bG41YGk`^|1LP|0u4lm#C|j1Jy?$R|#a zKt9T0y5;c_T?BVe(1F*|6pxdmZCKuDCt#umx_dY8^I7_Yg7B!I_wQ1w<-S(VFetnK z;{GQvf{(ueW7C{+4IFvg6sr2>?N}}V$xav;u z{~(bp{8x#Dz@h3~Sc)U9&Q4(Z0&;K4Mn_=>apJw15!Zcyf08_DRNb*4T((BI>LltL zChAFY7epUQR+psvX30dwp6h(_!o>c>ui77*ozZ{8alw-(kgE;_m;dRCPHiRCKkCat z#mp;8(Q?JnE;3;rITBKD%$dNFU$UB zU6@xE=TA)~uSg9Ytx3#N7tDi%i6NE6(q?fryE5gyb?O-es3daxY;^F*9wArC{u+01 zC~yOMC<9{L9BM6>%>lCk_-e@wW0mRWg%>8FJi(}^lKZfh zSPompOB%A;?~6?rTU#S#BBgN;D1vIm-W4aX>YqYffWx-lo;`3nCJx>a7VZ)y>s5&{ zX(s4A(YC_E?Ss1e@+RhlzdV({3ulYa8Eq$bZ0@~5aOll)#IL;3INaXN5_BT>3%ppy z9DXd&rKG(O&(mEE#Fej6@G3=2(F6uuereJteoZL~v|?~CdV0NsR+FT6U{8Ba zZWX8|wAUat*&g?d)n?oENp4lp4@`)|Oevc7T5-3KgU!pbsy3Hvjwd?4Q zb6BbNT?ZhH>Yf95>UJw=Tf0Ugx%J@kYz=*JQaSD0ElX zO)$@b?(wKN!U5Vs& zB>RYU|M+9Rnrs%n2WJPKmzDT^0$xyT2e=u{sR6e_3a1k|;^3JpAD2KytnB9&N8Y8@ z{X^~2q=cL3Uc(iYCilrbHm2EJ2KEtmqE3u*gC(NE1NJzcCCCy$ff~>YEjO!%%<dEK>G>M~ zA)^2cdSbULYh3z#7vyM>XrZPijEgXn`anm7kEW>QSN3C!1ynOmODAhX3sw!H;SGpk zl7bC?FZld2^+uGyNU=!+OL-GYgLcr~$^KViyjoKT>TUNjYL}74Wb~wtrtLzS!W=9m zkSkBX-s4FeG!M~U_(gRN!SuN}L5_+bn2a_lxJ>C|C%T3#yia4&XG{>LMo{*B*ixk^ zkOFIVIWYBg>J64|bMAH65jjZwtSJqtyp14XhhrenyJ8vgpAAmr@+A%lH@q4>cHkZP zurMBiN;UhaU_#zM)-It^yW6t5Lz1|4cYI4_UhibPx?t zGQ(jf0VS&UKHpoN0Fr>XZwwM|Zu`s?%puWIA-@5+kAsd7xdi|* zV9Y846pPLSAXbwzY$yejrXUvNEh4fEzj9@o*`iJDjP-ej2$*O{U?L9MM)`tl2^&*K zW&CV5; z4u2xTcO1kbqS)p&gfV_5yen>&4=53{d4!qH+Z?HafmQAc+Oma`AuJD;P=N1!e_R(^ zwJ_|It($2HNmEmD{V`o|WOVp=ul02mJDY`!?N1;f2yR4qJzo9u^%NMLua; zu1ZGosk0}pdMK=ul?Pj@OkuLd3xrIyQ%kZuJ`H&YOI$baTZyr0EU*mE8tDDLBSXpV z68doil8e!T%@eX%EYzC(2|H-&1E&Az@By+ht|jX)=@n#Sp-l*z1oYVB&OpR6WQ(90 zaU!b37a-GM@IY83KcKUhbX6qAcK!T zx4-c%-Z&SD|5W*yxm4nkHAD&E*I^O>ZnK0>jD$aaPOaOx)o-} zi=HL(z=8TM0D8orK6z%GY}=5_i)vj&ecY|-ObwRui@gJ_*%*P7ry%e zm0?0$CoWLEItmrJTtzgL4Lfp}c8PWM?8#X=;Udo4-|+qsKk^@GQ<50c-;R$NchixB4lgKk$gum(~<{E{LUzP zz(!0pVI2C-z=lE$KQVQZ4vFa3ZV{CQN}NxEy?M+#_dwt5+mKPFJ+$J0$F&Q6{ku@g zAAvD{ei&sK`#ciG{PZi!F-K^yh ze@+%Zsx6gLHR?kHFeD>WzFHc|)6<$!Y)TYs*JntV5*)XI`{oxPvX^wxcvbrDSwfCr zmLx1kOQ6Fye;IoR&W&=QcKAY_^byC5`O*nId+dI+=erARLYXW~^ipy?^FX#iv?};rwKw1}WCMgHr_f zX1KZ5uFaf8RcWm$qzsf%&n2tO*CEDw{wyggR16U^c~UCn{$|f`p@|pHH{zf2sF*e8 zNBnJaMWj> z#_p98rQEb{Xy-%X=G}VAtDI3s@w~`s$TRlfXp2AxWFOy$3ro=5df2;_#dxU3iUbt9HR7T2^>!)* zvP|eBq?(VgH)S{b`x`$s7$+6q}$E&QOFHq_?>2bdot&i6E;dXq)|~X{O!m$i@vr)7KtQ8J|(_`a9Z=>lR>Ty zNQ#3G*GIbd>$j@iW8xwAMHY(yw+(Q?3Q>k|Q6lGf1V1=l)k>I+%Y{!SQ!I@vN!gRF zrI!YA6pZMqs@1TvO0;Wp`|vrp@@e(qbXs2Dwt&2{VF$e3`vrAjtqA6*g45S*+#597 z#R4>XF1-_pVY>Pr80wj58RyUOCZh$fV!{@xG(6x7i7X;&@4<;f##fenA|z=p=T+8Y z$C|)k=KTtCjDR)K>66a@3jZk&I7;_XBcpWb_faDt%k>RRi;Sf54^0;7&GUSzVAu`! zWj;p=rbJmS7ai_J7|$_QNWy|aA~yTF^<$i;KY^+~Am%+Ftp$Y56$MLhQ06^Q8sBL~ zb|ga=GKGD6Cx}=W;LkH{@XvcvQ_6fF?Z_b~UMvJHb3I!wQYm~MZ(ZSX(ayY|K+axT zkCLNqfn7F_fLoB0Q*0rKMkG7$rV7Aq3C)-z=<_y0tmx&BG&`5PZ%OHAgI@+_~u1KKR%owdfkJ~^o2XoWRiEn12tpS#GNx>V- z9`WDX&}i0&Q`(37HyOypBXu{q;vB0U?E2gQjzNA-D}es|NVp*#^`hEu7 zAp`gT9@tU<51`o_^p4zvW5Z9J+0@!@7o^q_S0l|4LB9pK1lOe*sZxjTyuRg1ANa3q zn%5<`Miy9`i}gVRCB$BU|>--3ajdX7%P_agXd@2)a)`v z6#lcy?^e~sEeO#;N;H{QhXsI1YW>!?7J(+?2$BjV9JS2`iIym`t`u^KGUFDjO062Y zzQVo(NF>9Z(giB14;Wj~A;3D1^$`jOsT}4$VzoBI<3ep+EI%0uTC#|0VrCm!r%!;~{}`u*Hbhn=C?qhKSv*>!`Q z5}wUV?V9`SUS@AKUo&YRDcl+w=l_ie3Q+}&aZE*G%|O1HV}`(dloAMS(Xx9GF;iIj92~y(wz`U1G5d z<1MkcBI zig@M8*Rw_e^Pkjj*PWlmz=ITY{tl9R*7UR^Io-Eh$QKJ+0r?6_63(6L( z+Uh9M3=Ny69#4Kll~rKz#iguSSRg#N67tkJlS#%0gCf0rW(U{&g{Dt!CxnhkA>Y-R z7MrS73eFG^R&&rAvu@rao`rl?Q9`LVqDuY8G?N)P&1B~0mpaTb3P=sV5#H28vG0Ed z!1+aVv`5ZcE-}*yPpA8%KXjzm%P$q|6n*2el;yIt?minNIg$GOvk#Xx7#LH$>_0iI z8)j(Im;Aw|@TmX0x1bCwOS;8bFqK=tEIo368x>KDFO~xm>n9~1G zFfGk^`VFRQKrn^*r2fn}(SW7F2!(e$qgV7wh8?}n$}B@t^p2%JPpPZtuhC}GpQBA$ zo4=1XVL<}G>V)%+UWx@{sXr!7Za{rmg8zZ<7!ct`dfb0}Kor9kH;pKOFjxpfzGy17_+Ga)box zO{F5&WE@nn@BK-M8k^&eNWvf|)ZiY!hn;TZsPeB$=GeAYRk@CUhVSDzZ)FCZOK!Bw zU1JG2+ zYl>-$fBo8%#s@AkzUPBw%zG7*P4fRPG{V`tf^&!!5yb+NA3dAJ-vNu5q zT|fzlp^vmRGgNR|rH0g~&S2w`rkV9`7#cnd#L&_IfuVt2DnJY^#0zE-!<7U^t^&1p zLp!p9?m_W~naY?+F@8tQp4k3P)a=#P^tB`^V0fPebL8(Js;1(usPEGkm<$d#djxBA z;7%>3QkLxIdbAikSigo#{5Wdf%Kg*U!gWBS&6aZH&!CxB@F3suteVQQ- z+n|{j+*WlTpj_tsg*kx=ZJs=SsoNA?bA~_ z*h%(Tp*PwL`I9#J9ocV$Llxd=Gj1J|_ZiOQP}(Qt5vV$2*sql+kp5vPUVM^06Puo+b-n?*4~ev2vG18 z+TB%48o7!Yxx#yw+;$L0dbQ$4Q$kB<1?*H=wAj#kR7bU#8PO|H_*j(BYZ{9=gD?C7 z9QNE27>nfg;uY9Dvdo@4vxS3OsLPgYb^%_Glb?a?#y#31&mG#9@wIpLYy91v;5T{Z z&}ZPim$!T8sla>U+^nJEmU5eLtadNF*zc}RE~EGKU~qtk#~|d=ugyrr^p?OahULI5 zhVPPFtfjaim9bN%u0(yP%7Ci-*Qr$=C}l0r#t29>8$OwQG_o9N;!S4$*ZJhjzRJ|Q zV4&1zJHKhV8MMp9qG?FVHh{)_t}1Ky=K zDc~OD0&IJYH32q;fbi8p59C5!J$)B^u!3w6e!Jpg-M55*&fm-K+HweR^Y#`lywb{z zg8Olk0?j$S($PcN@*eayLAf}R-B696TwN#bFJv)%`e`JT`;Oub@qLzo=Yu9^U;&5? z*@VkGDmGFt*lh%0vKV=}Z93!>4Vi$D&-0znofH+wIbGZl?51-0GW-pvV8Rz0f;}1A z&0>h<;BQPP+>cKzVci-dfUA#$;Cls+fQMNW+qwaCNHl@9a{u;WKF|YUp9jFTIVH&T z+tVDRooxib;SZ(Mv3oo6c5+`uY^)5VI|9~#1yt?Rp3!)++>!3aP#c37EjMq(W@!0F zY-RtQ*dG21u}xEF{f*d2{vbA1AhCV=8?hb!Phxxc|0g!FH)2D7avd)I&&1}#ZOL|P zmUv$d+o}mFap2A}YxBeH<7TK~{aL1FgLEoSe2`_NX?jWfp3^g_u$~ZF;5LuB-|A6y zBg5G3n5H7NavzWMgv_aKYo_pcn3{j8Ea{+mywyy>{|KkK^rD+%Kh{|L;2GZH=~-PO&HFb4WAzvo ze`Y2fC3fFqIO+oc};bH&tA$S8E>CD!zDl%pBq$Yp!^me(wk_9l-LTOwc&eye+qEa^$|~^O*mg(u)uX z2{$B)XlDTW6(y(s<4Cb7%nUeEj0~4+z5IGfz{Q71_70DLg^N6UL~~?B^9by{vRI>Q?RXl&(oQl!u>lPO>DJCtnc zlw6oTnczusRV_ONKYH~Rw(iS>?xu~Q!M%6+PWXMxXqd~qTd70F1n)$q%5n1ZilAw8 zmfl-PuQ$yQ|IM;!P4HK-1YUT0Lg91l&egF9&8Lclg!h9i8-o8PpjYwV)o0Fb`2Tl6 zkDdn@&^t^S4&(pN0X;_q#Xnjlq}oI)5EC}|Kf*#jU|7zL2oCUGs~>oW_8WbUgrAl; z=#8*1%JSg&w6aP(VCqFZTirNyS=feda7_t?b+vI)-ct}dl;xR zdq30us)LxUbiPaBr++pA0FzM_FqMsGafPX0%M4_AH0UUmA6ce*rRz)owH#qHyDcK3`E+JTTlWzOBB6 zd@llf5YPkou@CGiqIh;4=sF=oXbLgMWe78PS*qwSxvH?Yh#K3xdLlSy0C81KQB-UJ zD*q|Z^0w@{flJkcITkL|>Dz8TSeY$lK{KbwN^Iv6BM@~q(>YjP%5*{K&-uNK(c-;s zEm7e0bBn_SGT6`y%A5mz)eVoCXG?kT5sRvz2? zX1@xcmtpi0UB%IU_&oa11;+l0j{xO=C$YWLvK)dqsbW^lN42C%!88nT*> zrg4rHMM&M*7_b7;yhhNhzM=#%(Z)j}aBCfK;cO_)j#$D0>kfH>pIrWh^hJbu zF6IVKhcUiV@)Nr=dJU@^vCc~DMCH4Cl6}{;exB% zA$HL5Zd1J+7`kK?N85;73L<;qVak84fnpIBd=D zR&gx+2j_Z^n!Ny8V^u9E;)Jo*j*nC6+}C3Q{N;XE@j6!g&JHsAU~E~={3qtTEl8sf|c zB2lu-;t6xWd46$R-DNSlrDaie4vADfKO^lDhT_@xW}YN3H+s?b(8RkO7?Iu%d@F+p z^xIz6c(YELa!BXne9^_IRbvVf&DXGenR5;`M%B%E#ooVsIz2F?a2R7$%IhfQdz%VU z(HLca%{dw8FP`iXF)&BFis?yXc2m-CtR^ZmX;d+xGtvm92=fQlK&(*yT9)Z8W970h zf5@2@jfFRXQACTo{0{S3eQ2!WD>GBmOYuFGe{Y%wdue$kg$Ac(=0ys61Z3q;{W|N< zb=HR2=x$!g$y$N#HeElVe%PV@up=^w0yPu$5PY7kTyPg*>|j1NsZqk1_!uDbLlknR zNkR8A&u@Ak$5c(R&9jeV%6u`%+C2WtgYnXQ9=^Gs%Ed&G*>t6Gp%G4fU*K`la4r;*e3G2!Qd!e8|fRVlK6^qB9-p$W{XAGl?kZzC&lztJD#t7Y;3!xRbe zE)>Ge1bb;~@G@a8S{IMd$etBz4=0Y;5RvPos6**h`UC#7rE!Xw5}j#;bY6P!Uwx;e zfA*c;|NML3DLSx{&Z0zS_woPsMVhJo_C>NT(GEj0?=OKD&2 zy6*{g^=Ad`y&>)imK(};hssxSl$Jp*G747wlTQ-oq&*kWk_0pyr_}wIWYH?v^XQPN zY1+OFUf7Nb%t6>`g7)9lyq4*s3%wvltEM!uac+Qdc$l5FTIH+Cs8oyS`@MW}-Z|x| zwx82Hbp>v7%wVPkE>5LWAp_EgTa@?>KO)qedD=gV3Q$E`$_dTmfokzRDH9c2jz6JA z)Ukf){+&l2dU=Z@7vHohJac9TUJ9bC&i0eD@*pO>C%T+0CVUhSDrg>qGz;6QQu--? zg2!$m9z3k3I+~hYTqDHQbfA1g#`&XS2?m=0kJ$pL?*o^Lig!xk4YFc0JPO$lM0XeE@Lm%}7OW9?NPiipWL&Ib7lpzr9|b{NDK`^l z`^ANTnT70cZT=R^!nnsDo>CVq?ye=*z4^R0g`{YsD`q;JU;Hyc{JP{c>JO1?E(v%X zLw6(SI83;GU)B&_kYm#`1~qVl32+VZ!)4KeQ;ZxUv%ee(8R%{M&mkbdpOf8Xdm-a} zdlLWB{YumQl>)A^2juvv;Oxp{1$oD5&tYSzQ)KWWnc?;M;up|t(@8{*?EQRuQ-61- z?FL$ug|^GxHJ&P3+*g|j!DKcE9O;%Lm!{9n@Y!7^%5=%6TX#VX^`_EaSI7LKRB8HT z9TzV54MmBUnbWd~GF*f{RG=Kq9ajt&liAk2)IN|_$HLBj&Nb_RE0t`@AX)PhO%yXd za?GUD%Mew1|Lmr2NneXvnB(9?seXoTA)VN@0W6K^!Z9*S#?zW+3O4Dx#8P#sAA~un6ekg_hCW8HN!Z9% zr2CxlLt7cS6wV|-a@~6<*kxPwp zB^F-VHWi2dJX<|VrY)miY%%Fo3nNJ3v*Q~YoYDqq$5n;d*o*iyG3Tqs0TIja53{8! z0NvFe$cFVz+G?uIxupl5&^X20fzZ;JUr79RvrW6W+WgA-{Abxya>*@r`-`jlF`L=C z4UrgG)$)c^`8n0-Ao4WO{DFI)&a7my%M1yGGz^wd_!6-svcf8e+`f)v&F0j2U6lUB zubfkkRo%AXLC*_Rb5=!LFo|!5uAy#X3Rt=CpEdHc}?v)z+tN0TMv6h}_tJ7hL z#3OYW34M;5maC{-)tt^`GfmEvtF>OkNf@`Y`uEk$~#_u_NH&AJZv_UKvN(9q@7jk6sT|@CQc>5m$f6)9j)Bm7D|_H>ZUqS|>rg zbn7JW%eGE})=AJh2{KCGnUf%l0cgDht(O3_UV^|&pq!|09R-;MTSr0G(>e-D3b&4e zlG1mL&bp3*)Z6=YzBymGZ|;ozy^Kbw6BwU$HCG#r@*q4-m7-BbD>#~p4Aj=q6ffO6 zn*6e@qp5W?wT`BY(pwXaV!DKb-TIpfaoqZw{O6(jo03r};DwwYt%I>@M8N@1WtuDVaC52nZK}qSmMrU2eLF(=OidK2)y2NchSmjX$ zHu!?f*Fj!m&{zB94J5>lT8|oYXzTwAT`F-i_+I_U7IGw}?p~^GGJTeoHX7HiTnrhc zyEeK(Sj+5@LEEu(hVU#U;WZ0UmMO(@0%jt^yuh2p;EkApCYi`P2>;9<9`6q!EWRb5Sy_85KpkqD34&GloW%BNfj$riwXH0*`*80 zpk0X()$+BNv91%6vpmAc7m<=#$5Lsjny#Q5nO&W{UP|T&!iMFU@KpFh3sRX?m`k>2 zF;v0_!F&thUf*1eOY`IJ-V8*SMNtFcA-ASK@d^xv~&!!RSR7A^$#Zv(HhNRO5 z5#CC#WT94i)mG~ri!AKoYsgj2J}iFUC0ed``ZnR7zHjnQOoDqQ>sBYlT}Z+_y0?<> zUP{7)Y9{5ux-H2ql`7HJVinnvRREMaeKiU#xfx$AiXkA~D3=Aoq9Ofw+1#Q9F6-)$ zSJ=*|#B-n$-!%qy9fS{xHlX5g59S4MFSi7Z`m}pscXNSIR)f5}_TQ*p9Bg{G3&QwcUTStZs3BDMs|%<7=7m#k@2yly6>3T<#ok1_p~?)DfeWb}v#AVQ(J;B* zO)V#ykWKuXmfNI*j7t%1muT3h^*9(>_B7BAGWR#L1z3)jqYbjeRJQQa2HCGi{p`}2 zceV^O-ww?9O!_T}OJu`wkw~ZqaW-g&*`Q@ZJj|S_+A#CZu%pbkqkeMKxs37$5#fj0 zr;D@LR&cP}{jYHM7wmy|a$5u88Q25Aq}`XACqloqqv7|H>Cus43d_?II`PEU%u=Dh)r;N`7(r zSD)%w-wVA5w71Z0t8O3te!qXXzc2pV@As4c?H?VD{xaC_kA{bb2czM^U;4wr{^8+Y zKz}_$jz1n{y8D;@%6%mVcTXP9muL!fY=WL#qh0M#l06-{a^jJ*GWn2031^9}fzv~?ZDC~+r>7MLG13mB+AZ$X+z`pEX$Cxk? z-b0Tc`Pi?9H^YKD_MG?l6GoUa-Lhnh0L-Cf_vn1jg;KfQb%=$G#WBz>A#^lF_^5vh z`d6C&6uMW)fPR_UGN5Gs4~Iwn#QY!b4+ic0-^8=T{P(%xtgxM5mf1?`MDH*$p_D;N0h)sDAhc~_p9OAM}QSEbM zj~5W2`!dmI>=Nb;OopIVc)FvYtGv33Xyrd0oz0Jdwr4};B7#Ip9m)2Ig`yl8DZYEe6FsX}I`>gyFuK&?| z-P27|Y!K_#sRdo;bI}>WQ#32H3QE@h!T$a*vHlo-` z1W%k;fp@wMyUCZ}_@r{n0PKQF{z@(|XFxM=`Byy74sU_T>+ld+xU1AN&D{;F1f@&6)H);rr(h-?n_H&_oF!C0!J!;L_U_-x|?iY zmWVf+bU|4Z#u>AG?ra;kiJuRXze*UYaSp1)GTZ9FtiS$$G+f<*)hs{xl-hs&qs02Z ze>7gunERQniy2OG_ z$A1hC`-jQ+kAwcOmH(S~>dF6^MJBq{b0@mdGhC?4d87(>@k#gCoVe%@A(MKS--~dA zQ1qWkR{B7!oS3eq<$sexZUwpKa*hA^kW`4@=^f*39T9VRBEL5#p70Ir}r z@juNV6aTfi>CFGQ)|oMX^jB~(M-({FwGrh=2f7PI2pPhF%^^qt3tZ@sM}Os?7EfIW zX0AS&Kod+|VuLwj4n5x6n;|y$Cc*$wCcX0K#3GYDTjzw^Gl}8Z5X&WJ58mjuV?nyd zNxw@W^PJwowqnBvziQfVIfGZw;JD;wA%6SO0rF!Q3gSnCm=U7Dui6YUt?TDw>HRkGecI zf(`0TK0_l2q>F`Ko*Lq?-=kWNB!xd{teJOV_m5WcDJ4VEzFqBn{ZCc-@2`aGIR7yk zrS^XYhl5uC-^jDH|0iIGpol+ZpZFp{Y+oFoDAB;XRRV$)Db|VQeN480PF$vL^`wQ8 zhzJI|K|Pmv44%7iif(v4uDw-oC#bc8mK>eUX-?k(R%Ds4z(2(i?|xTtE}iL)K0y{@ z2-4o^`D=e6rM;hZnxUiaT@5CX=>sjf?&g-{3s|sWp;n00;aEv*3h?AA@mh{XS)TKK zV!_Dubu)4)4yNn78T)AsMi?ap)tul2>&Yh-cURMg8dD>b2= z#T%}zqvckAn3|d*O1=?At?r1ca9Is)EnTtI+3xJAssB09b)jP+L$BcpEYbh=hoj{F z|LE|r_5W?;Sz7-KqzZID_zY3*EX*hzs8C`9A!k8KYGASnDK->+uCixP0VD1s^ud%B zi#-@o8!5Fd9`l~LwN+XFeUga0N=?I6oLyqM31jku^$K;t6QQ(wm?0~mj32)o*FS+0 z)V-1?Q)+cpyqU7g*dkn_G6yxnC9NQadTF}wWfg5EoVr3vjTC$<23AK@j?EB7(G5z4 zQT3Sx^IBFyt@FA{87=}P*~Uvj)d~=(!QP>=9cXN!`kUy3zYmSVkQ;j01z!+bM>wvi z#r4RTE*ZcW(2A##CSDL`bR>qPoY(-^F+6y2Kq@WENXo_e^-TN0~`H2Cj7% z%d!qK)9WzUyVBh~LpSD7PIvil&q3zjWkPCr1u>&h`eh#^gJ28|IkG{2xc_%Q=dH|5 z$(R$?2Uz%4$eGmriT zp19B^432fvboqP#r$?d7fBf!2%D~PO4$9btQ|Lms^YrgQl5N4j!63%G2aYWH4EleM zm2#nOCW`P<-zSPuh%<^E#0yQl=@fE_8zTalPUFP{LB?E>E2Ne98(8oPTJ&UaJOr0; zLC1d(3{UKZ`0w$c-?st6y1~#DJQ!q{=Od# z7XaPh>=^N#fIs{?`vW<#(c!phHV1>t#5aT3&RC2H3=^LOtH|FB-mpE# z;tZlYLah;D>7Vs0of|AM7O7Tg+;RSoK zbm$7tgztR%|2@0d_)52gI2iW(VALOifmv2^)E|IRzqFW+7vR9oD>Yd-I;8-}X(`_s z1T&XdY8Bly13ZM!|LvZOMdwTm^NHtB=0e@}8tPN3j_e^#OG{>7*$k5bUDYzxP zfw8u?C*+W@2%q<#_xJv;X6>imWIRO{gRal3f~Drnjtx#JMX&+Y$shk5y?c74yWsrv z;^*=ChnKHj{~tI3n)dgnneThr-=D^yLQ}942t?j86sN%d1ifef>FxY~$Dip}gVL{` zo!M8;>tAPR`qja+ugD-@4bS~L`{OIUn(_bt@zr+rzmnPPE16w=y@nIJ^Q4o=%MWiH zBth=h8TGh5SdK4CopS?pac*fLs+LsunxwrbQys<;1QMIPLWLw*HM4f*dE zsecBfME(y4!=(Hl9`uh|`M-&0Y5A|@qadcA#XCRUM>73aJZ(~yW|zsH6t+;j>$jD; zkeb}RYce$wDBdx1i1}6?d43rs@K<`s_+F67=`)!KK%&lrHOazWDq;L z+dzRfU9mJ*l)B_7Ez#lzIFt@CFLZYX*_*_+YT`o@pRr1%@CCh5acch!vXQdMro5C0 z&M0+XGF!Dzm8bIhABpJNM?j0$|KZ``VQT+pwBPRkY~;z;-gAt(k~@8(y+yGxs6PZ&lD8Ov^i4*^4J3}Igkb1wF5eyltui$6#B4?;r|>oZnP4%xpbXk$ z|6tJAg)>B%yO6w<;}OZ|q~}71P%fbtvcIbgs+6_C^jWEIC#xQv>%P}zuGi`?teCG{ zRssXwFzCjN8vx?(>M=s1y_NYO`uS3Ana}t$3+ldIQc&s`Xj-OdT&mA1-BqGuaUZD* zC@aD32RIiYDzC^ASjuavw4+M+DQJB;il@P|T^r;Jlq)1dWfj9IsF^ z^+&aH{&5KXE!5x!J3g;&${Xr;s*A(q;uT+SV$dkSEUTXO>MI<_*~5sXFIjfInPK9> zcS-MsPl>#*RYi6|av1gUqt@AAx7dt%z_4@ z8vZ;JGUC*<5k5VCU9pvTrCO{Y69~VKUCkMPR}qGELrcl&E8X&hWfli1{k|${G8Ovu!$<7;H8K!I9 zIwQ8j*D~M!0U-XaB2?*ymLh)AOjKS}MUT=%l1z2Mk%tOsuw^TL)Z@dTgL;cM|=+F6%<&O6ty_6sQ+aLEsqR*fO@F11i$O*Lt#6s zFGaGkUtGkX3YB=V3|O+I)#0Mh@La?eX9P2N!@^}-eOY_?j2$Wnc!7MFDxWcIK9snlXTuUac8%^%>-U`!hkl zs5t|aR#xYJ8D=B4Un7~RWDBb&ql1=KhwFC_1{)LN&w8-P+9h6izj28)$gK$Pf2p8F7e3W{ZEpMODXH9Ds|c0d=!cY_7)ziF`7|i6T^@ zzT$}%!q-j2`ug&6Ibu*UWo5*0#0kbFsBsO^Y|f6spx;-vr_B)8(6v>ht&4aCIw_BYE^0NU+N)tV$VIx`lvc%l5evFc3uOk zFBb*33aZz}#Xd^DY5p=n0517VD88(&$x%u)#^YS7jd|5B)p+R3%G1BN%}cmgoeQ$= zb5GzUqex>M5#D;A#SOWa*Eccd4^RlrqA|ydG&yEhwp=#$Shxu}d|jY3-lHrtNwLZJN! z2ED;SufN;ZvLLy=P2A)9R37Gk`dciXE>R}R!us^LWF$xsWg?m~1HZp$6+vD;yeCzJ zgg^Bte5zg@E7lWvmEnUvn7yHAN%>#v4%jmJKN=+E|Nh~qJ^!_lX9Mzo+pdS*re$C| zu80*FX|IX>7M~jOf2q4)|5D-puff4lLjL!U_S^fvH}Y&){^zBF`snL{=i>Wz-v^$x zc@hl!wB{k+g6aj{fK^jTIWD?F7Mwxz7)*7G@>S%o;Df)*3m5^8D{{R8NDbZc0b&HW zgs`3{tXmejJ`dV?85UC;yqF^j5CtwYJ#6ZjfosI(0292+8v@@6-y3fzf=KDHD;k-2 z3|vEC=omOIx#GqQVqi*KZX!WUAwU?I&|z}{Sh!W~5=S+ijzwc&7o5I%BmO)U{TjrD zzWWAEynOZegm&G3y3NDhCRJBbqqef6m8}AljTTWh&F{MjZC& z1p$~aV9Z70ID~f{OE=(S@EQZ+nsQb1KQg{3O_2q`6uFcMkqV}SODryxErc(R9|a?j z&?iAQh;4fqvBlmTTK29n*Ih>Umats}hWzsS_S_U5{A`RLhRbgkHC9lTtX-${CT zPX@g~KhMkCNtOR$&>QUM7Z1Sk$M8XrDPa%9zDYu=Ocen9B;byJlF7<91qW?K_XMk5 zHqnFHUertIl!&)q-ni<`=p=2W%9)lpjOfssDZD5xgIva;Yj?6?2q1bif(|m z>xV7gZH}hq5;Mivx9CFOtb$edm}7P8=6}kqT$%xD7An)bKC7fj9V)${86?UagHxrb zj5T_V4T60`CWzrzK@as@NU6wsb0N)*4kc(xJNE+PUYPTKm)4#cV6(DxN9(eyQ`t^?w??0W7-@JTvvD1;P zKX##5*>-IGW+!6aaUj~poQUHfNk9abDn`p->iX9#SkOI*OSIb#m@L2!FWm*7$)p?2 z=Prn9b-FPr$&(`;@rH;K0sFjY{DaBWlSPFIsx#Q?WM{%#0);h z87_;dES<8kD}WnQPh6og^{f@x44j;VCZ)jlpkI#gYu&{N&+_s87TOc&(%LgTHg7p{ zx$ZV49yYo435ak4*s}3puKUwFjywPEBap~cH}(sU{WexWoDxU=_yiCPvS(*lOkRh| zXGvIYXHB4WmYqM5%rDjd>)C&h8D`V@KL1gPo{`FiOJ0o!-+tUC7pNgbMPROgdmkQDLZOD<3>ZXdi!Sy=P+OekS$A2_Y8f zM=A|*l@=oXhU@>(&(0g~CDu<^rh7p;V27_FJG{5Ce~J6pfzTaOh}oE{*q#gNm=UI1 zDcF7;*}ru<;78E!^#@9K5v0*yKgPA)(X!SB@B1~Td}|VAyF4mG8(KanIiRW=`5T3%mwlS>NISmH;JzN5Psh zyDvBG$Ffi+UgP%EDror`&F1es4*vV8M9`T~WY)rGc^u{A!`Gpx*Z@XY*k0Tm1%m@% zBkVCqd*GG0BoH$oPU#9+YwHUzfj|co&4kA$#?N73Zx_tbY!3X%5BOq+jYSs_7ntGo zO_cMXv;qxcI|Pe+Ww}~I=fty2-VwrAkTNvmT4{oY2y;z@!kwMCAXI592Rrr$JEjog z9I?~ku=Stm6E35cZKLz+;J`i}*uQnak0)SM({jr+#FLC8oge@BaDMhd-r%@V?)e)h zxqo#e&GwSaT*nx56@-`()A?={g2i}%I$2AmasKi*==Z@hArZxDBl-=eG&U`0W|OH4 zL{Wp7@bQWE{$JYduLoku`RU(sDGR~#OZq0h|4R<7qpVDOkCX+mCNQh^{bSs}wzR*e zWwVye3fb(S5jQReM`fyy7P(#rW6eICpG6GBNJakP>NwLc5uW9;CT*6x=7=&fb9Ha;}WtQ)?*6?o@gG{-9^X%fx2FW0lL6~otU!S z8ywV?Hu-!YmFzR=fsxl(uZtagrO{CPlA&*@ninKu2Sm4p8*=O^mw4D5GZ#7Hp7|2C z@OfZGg4S3*&RyaVs#}{?a2>x_5Ez;%9e30Zble8W7m>!Fq(T35ESIA^i99$EdSYC% z4p!9@U(p3NUie#DYeD?;=WGrf=o(OPD3~s=0nDIBAw#!kx|oPwj<0oVJmq*92VgES zQ<|M9TL@ZKPbN01*JXz6-}MG{`>ge;7ymI6VFFh5(0}oi z#Q*R2kCO2p`~6Xy|79ajPW(rV&~4~{8yB)-T!_z4+ZN&;WAirTG zRt*!uId2`y`oM-dW?U>oOg)#2Wt|C>K+KQ}!z-wPgB)lf40-LThYgudwD(Bae-Mq! zX2l=Z`htsiE;G8kO&3myE6#-bTgQHp3G_<=gk$l@YU4UKl_nw3XF4|kfDU{s$L)a& z-iII)!VJy=7mDy9U+FF)o(OM7Q{TI+wr@keFd=GFi37@cRgV=1*K>&F!WE8EL`w_c zd*Bsu0ld*|$KrT|YU|nysGDzfOh018jJc))$VYw7hx@`~ix@)Ve0C+0aPJH{rHq0J zhOJ%j(4Knw|K<0zrtbfja{d2kG#n=LKMqE1{P#wl9Q{8ebgTQfy8nv0|K;~|+w}gB zy<46Cf$IDrh12?e6i+MMtzlxEvEg`QkL$FdlQnX23*1|fXqbd+#=KYl zPK~w*c_C1)Txby2JXTn$wDV)?*<%i83izn14Q=8s{MwTRgR~yVOF*0Mk^>rpF}^Pm z@1hGFVvgl8(>Or!K=ZW`DC{G5u;T#2alDOXNtQx~g78FeJs*kaKI$q6uN?;)5dj4f z+W*@*ha!jP*AE`%c4p^^jIAD2A%VbmZEV!y#HDc03D4K97V!}(O0ceC zIeySRHqL@i3T?qZ@<;MS>cubHGwWexXB~;0zkyIqC#p6mBi%KJsr<&)P!VQ6IEQ*lQhA z%RM#yKb)O(2g!KHs;;2Y{olh;GXL9Xzuo`c$dl*)$sly=3Tj<(L?nfWe_{hkyL(;Cu}G%o5u1~EP}DnW-TlidXcUXn-h=i zL?$CwpE_U<4B&VD!yG*-fa4fE<&Zp0Xiq_cqOe#2e8=h0RT%u%jD6!nKiHO`A2dDm zR>6Ld&l38-vV@3RuY+igTyvmI{~r$eas7YPA0D>%KW*eG(*KJI-I@cfIk4)^us4B` z;{J}9@HB`F+eYobcs{hoz(Y0$ifNx4RLXh!1)`V!Cd@0(vMR+?zJOOK>v)nGzL!y? zWGe1j*$vX`EuFeHYVsOcmWZD8EK5BquW~RNVE`fc`Hn%mM?TO#eR|93=Js{ry(|-^f!(|4$IQH2_)z zpm77>L%?JA!U9Olh}H!7mP~*I@k;|ug+^sKULttWmN^)ZX+f?Hk zyo`SdO$crL78-f3kawKjXMCt#U^w0~94c-zZ*rj7+Z|D#K~U~N9K*?SiBMiyBRp$v^W=d-ZJY+i{LG0z@g2INza0ti#FN9})AjV>BBf}*bNf0%ad^t1bGnu># z>Am5AqOyT47gdyvI;C~pnh_VVx&43;STwvQ(Ktckw9o^>Tw+8kN3z}1*?LEl^`Xqw zJDsZcc&2W6q8{A)iCP}8xt58Vjkx)@@zL~T>Z-m|B^|Ifvc zLAooi%c_(Is>J^{JWB5W5BK@2*8jJGr@;RgBXsNhYn^`$IR7M1oeGu;fPNBi$Un)1 zqh=*{T+o&lwjNbzqW$P|)I;)O_f`8)+lzXMLIz{)mp86D4?Ei!RuL04t?%>!`%Yt& zRvo7apw$@=o*cftjYsjSNnI?xGxt3s`9I+<~eR3^udnB4FF0@GZQoDmnkc=xd~HT?2-MF=kOOiasYASStq>6#InJ}aL@ zFINvv&tF$YuV4PI(8~vsV;)5Vlm;S*c3VQ_K8zu3=7sGU$kR=hhz)gFiVpw*PFCfsU{Lky_LDA3I&^5 zupW7oig!vO4u_v*tJ0E^Ncy%mGDQ06d*3?h`N=Xr%3Onmbg^n&ir|;OODb59RYAT5 z*%~p9i3wi@b1x6M@03E7F|1a)D>U=b1(Kx<*RgYOO=3KT%4p1$5vic*j?frQ%PK}d z+>Nmb6QEdvOcbqo#xAp$lPW_ju_*dH=a2y)Cy`_YIr#ip0y_1d;7NQd5ZT59L zP8%isEqJ;eCqFp|pYP|yJactAe?y29K#*6(7tnsX>f#DF@#B-Oa$OurqkB@8S?#sq;8E! zCCH8xw?BpEmXtvv1ZQwy|JDJ|z+oRe0|($o5WjkFUR{^VVv2W!eSo^T*dRl2U3&td zxH?V;nBZN$crIifZgHu(MevNTwsRc=9gCzR%~BQ<<|?BO&jx1P9+GiH_15asxf7{Vk_ z-bE;&>Pp=TVG|kLI*4v{R6yZ8yjlt#4Kq@Tq^123?j*8)M6WsRR_zloyNM zIMCqc3-;^@bOD)y;I@qoNUjLVpwrqM=s+X9A7mp;7p{r2Pnq9|E9p(L-YODOxtU@q z)>G;)$9u~C6U{N-U>!P<0%(x(h)NgJKFlv(ONl)S`nTy@yy@5d!cG4ElZW1OImoocNBgAR@@D#EDPyxNyx}#}|zdr@g0ZU^q+|O`xJ8 z!>x(*3iFjg5G+DXGhfo#wQR2C7|*>$^nUQ1D>Flb>p65Gm#~Y76+Ooxl$(Bp7h4NM z+K6T|WTDtoYi$$x&s>84NhW@_?Mig_t1L%T!zK8$tSl*tC}8>}r>9K>Ik_P|7rVy9 zb}YzXlkv5(cQ-a#Nj-*axYGfX&eJa){}o&X-a!7xqfsjVmSK!~lDK`v-ZOdn|A=(CvSVB6Boa{mw^4vyE7RMvWQKcLpBKLK` zw?aQV&Iq*%1{7Q>N=2${K9lJ&_3TcjTP;Yd?I%e0>>y*eTKvbz3oVmzXV=&rbi#Dq zg>4G`tAZ;nR~vo{?Z$Y)hBk2*Tfz<;n6B!i%R0NC#yVS0h7#1;-b(dN2qYTa-&Cz|1S+U+V50t|Ww_ilEtg>IFn za1(Ts!z&|Q_EgS1CAc=C{tqAE%NyxgsYKLws=uPll>rb19?cGv?)UoXNNKTuhZs#a z9`Z`qwjfS}EF#*2aa^QWWFYSOTY^bE3eET?8bC8F^m-*-;b$_D6(Wi+&5^=q9p{|q zwfD_3Ip^&B)5Y8+j9CcJenzuuu|H2M^8gDQ=j^~GJi-9iVmvNLV&VHak^M>UDh5tg z>^NVaIMnvvGy!opXguqYc-|7zPeBo>dTsJ0mgLH`D( zE^N{>l^l<_LDGn$u)v0~AREhyVpmr=^r9N0L3*GS;{;Qk8M>nzh%FlF>jyB3DTu3t z1SVPPm1kM6@hjrFFZ2bi#F;BC6XCSMj?dI`yu3Hawui2v-a&;5oq^ z`cvY$z|=)WG$f9ryM@C*$3KcmBbp99se>7lAV8DF|CajQ*2zC1*% z1LV-lCs_P&n+$i!s5RHA;tyCKvqN>jN-R#YT>R8`aBN$~Cw=u#Z)@HBYiTfXp_79Tw|NhZYYyWTLsbK#LLbv9BYyLN9{tK4cX3Jdw*c$HNfZ;AE zU1+zzBPOhsj^}CGEFmV$NVMZ&nPF#2TqV3c$WPJW!4lqoW^Ns!S&7}*;5ZS%CMrJG z92Pmz1WvTBtb3v{*Ml;%Wn&@dzI3Q{>i$9;+ToHc_ww&OBLEWX2<|nvL|t< z=wBg&h=6l~A!al<6x5*Z>-U*?3|>a2h=0Wv(hMmATVPTgeANw^@=Ag`o_NSI$J|)t z1bqBwczMBB|FQ$^wyQ-=X0<`LHgs6zY2Yhx5yVEVZG368G&yDEpfZdh#p2%y zZyj_YWn)<-3bhJ0LrrcN9rSvdx+59egwNmYENMNr!Kg)fiZ#hKE5%bi5VrBT2J8WO zD`b;#TY20Q()5zrV`{j%1IMCRNm9Ig*5QV9uQ}pckX;b+R>zBTVpb=wm306#MJkcb z&_CK)c(Sw$42w|6%?dV$z(O`+d|?<2`pxF5XN1@~UO3TMqlMXmZ0tkphgEu*B~#e{ z$YBD{K)?6AF&dnnzkVa{_^cf5axsVM+~O*Z4xT?}bKpSNfZ}#g;9nfd5t)m9|AN%) z9649eRgQI*2Te75k&O?2Dr_ZnWome%O&~4(D^n4q8JDIeZ^7vXA5OG>@A-4BOT9@j zvd1TyMP^!;BjWL&+W*`6|Bm0h{&?~Am(z>0pXI;e_xXqS|M%~&XYb#=dwF)zc_R5F z#viL3=xzDqA@aw5UH-^-VxOMBmdkTx{!n;Bq?~1*;Jsjpj8i)!h%!)1nx&}hy(msC zTu;+9Rm8PIJYOB-lsV}2IO^rfTC9OpY&}?|1D$KslJudGv{5O3M-**Fc&Cdff^bbD zeuLa2RPA6ms*D%Hy;`uSeA9TY+?&8#k1v?A5a;wTWY@&Kta$lX z`T39igQLSl{Kx2MaM;FwY~sm@|4<3thJLi6A1j7__?)zD5g#g-x4|9{G}uEWa&DZ* zJNf+sG$6XF6LafV1PC+ePIUtUE#nfCE2P+j8lZD2kX+eaV8Id$Vh7<@ztc&Jsc}M? zk_i+f)MFojF?8M7tyuB%_+2@G$A4;9kj;DFX}7-)?cY-OAq|(b$b1xDxH_gbs1`T~uU zBm2tgeC!XG#Qs*96HDm-4B3^AeTPqp{y!QFhe`c^u;0diZ{#V`|2d&s<-b+_S5*Ev z8*QcL&jD-I{)epga|-9`{AU(1ElcuTsPaZorirN78U;r8dipIjTFeodo2W1spbzMqfU~<@=?Oq4BgV*o%~eOnWG4T`;!of zn3TMMggKEdqML(~9LP1YU`MhjVg#LeeteAREPryVb2(!g=YzVDshfe0BGSf$-7?r1 zfDCd)DXd?)5Ef*x2Hyd-!kKQ&p;^8BPu_G2U0Nwcb|aa#6O7|Gpz`z6)Z)0M7>Ntm zIDE3#SX(KH-vAbqT`Tod-Bp@^+luR@CjBzgKD}_L_eeyEy_Q37O>%G9;KdSOH1M6W zid-wMfY@RgTmVzS5GpU3>l{Sp%Aeg!u@X>sPy;yfvz)7wJ<{Z^Y;FRKotZO3IFlFS znj82IWL72~%dg$ZWw;n3=(X;Q5sAdun0Hli`Mh^>ZNsWe^Rj6J1IW;^N)-og!{kk} z*gOS$rLv_K9OGeut`|(hJT|?v8M(v))8P72Vajvc7)2vA5xwL!7_wmn8&t;DocyWb zD;xH{`|3{2-gjRq|L2PTb0hvW?OhRflW#_gZWH$47!}EuRqq&RO8d13-){01c z4bg1Qpt)&UM|RvgNWIokZ2`^eV9(ZX5=md;5O;w@K?m$eOiqynWmdH@MSld#`IL&Q zdj8bl4f}OC+W+nAgt=4t^=3-H{xPM>!X?!>wHdK4RBcCDT~iI!fu~?`QCbuU$kYDa zsu){oq#GHafT+-Cq{@b5T(PoXSB$L`^B#Ox)OV3s4v2?=@MtHe79T&xooFM4(Bo;~&A|1zSxDg*+So&O%D^8X)XV!SQ_P&9-2DluAFiYzNtG}w(c&fK`>PZC3RXwot0(BmQLB&6|Ie_ zXBd#usi&y0amiDv#HOz_Tis50NBkV7h~WS6pa(z5t6pAtR^e^%F_Nih536}WAmly$&HdOIP8OGVASW`Z1rg z@&Qi5TWIIT$(n80F2O||uvgL*g_t$ckx0{q!tA&&*5~rQ6RvDX!8srMP&|F?8`Jth^ zV50F}e0=efRKX}$9ZM2G#_0AmIVC@nNk|BtrDkjpgP>fBYet3S?Hymi+_QCz{)kFR zl}_k@o(n19IFvW^>6C#f@mxPi{?kIbaxE57>Otza$Rz={(ZIwa#-%R~OhhkZ#6_5! zximII88s)qrb)*ET&qsf;XIw+b1+=;rqwuog_UmzQf?`YjS#{n=^V6*XMpOqV?m$P z)d(EHB#8{oEhh0Vr*F=P>w1E0)kSXu6_;%)rhsr*AqX z6ie0-xzGFt^8~G+(6w%jjcC*>k|#--{gnyxj8RuEOEog^7E&bGepcD zn=2PSx_}EmOQpl~#7!|~01;rwhLq{HL&3yj%4Oz4rqXhmnAuOcBoz4ZZw;nCS;rw* z;}EQ-bkx;14DWh-W$T|>`+vKG-e7-E)nqGe1D3@99rg#w`(F?m>4ELa4%XmfRtoi!C-kpSJ|H_a>d>n;W;$RVkr7=uI`Bk(Ap@vWCxDCkvjZ zF-<f3*LsHuIEL+XP?MCQsyU&%kflc6M!yH@Bg!BI+%)C(xy}g-%S8aO84*Z%RCD z0^PCzVRPsLTQ(j@A79|aai?Z|h@mmc&&1nU0ddMSdwe2}{e=_dA0OU-Iv>Az`RZaP zIzks^OpPYgnn3F;JAYbp`P+S#u>T=5%m(j&8XWca2MPOscyKgm?f*?YIrcxNaZQV# zSJ_2G{5LS|Ul7ByA(oPVk2t$C51CLD8Oa^dz=71-`47R)=Y&<@jBMKAz`7XTOD;hC zs{yz<{y0>_651FcGPp$V%tl7_`-ajG40{uuu^-mKf_0`X0 z(uLR*sU_mQ#QF)#bT3G7VL$IJ%n{>2=*q~YjOYjzwqHjg2Tv@_4aD+`ZBIC{)&26@-Fo|K>+QeYeQ#BF-M;sHzvpw#F|(Ct1*Fo$*l!8i612qr%vy5; zPMmLcm8gSfj-Y>E^sAA`yuQF)^+K;kPNT;M|Ac|91)ZILDNfc)Th!|<6Zr@9)>QG)8kuf5?NylLpzDgLy4#ZDHJ&2x7ZQp8NtFn zHOXc@9>U`-{~qQbX81muG*b)k;}v=DkCKR?bk8}??{iM>3K&~AY{D|Fs!VG;!8v0Dg!$|(;L@w*F5Y#LGvgr1Eld3WYLkFcG1!j>$Bxt>It#NkIt0K( zQB_CoffmaWq~&riz>e;%`Tu1mZG!(-W^zc+d{Fy6=WjhcMxm zz!T9r_J(mvR`51fe%wV{$vj|Lu(m5E5gfw32LaM82wH&6qKk*OlEMD&RrK4s)dD!> z;o<5CO7DVor9b>M?T;=*jLpnV42SWi3jG5U*D3v@;gl4s=CD~v28`dMcg|NB5z5;e zwZRuJJ<=wJ-~SI`f=_tprd(?Ig_u)p{4I}!OEou6)B}h(XuLTKDDX+Uv5%2aN zV>J1>UrCO*%@R_3+3^K(a$#EgP4H;Ekhb}+b?pYFv_;T+Z6oT$=`y7-yZFnvNzi2cjs#8)7xzdGSeVs* zXXIu%^+Po?Ph6-aSh5v|_`RyguF`BY12(ei$C0>BGz$`nOJP`}tRnYg zNMza( z7+!*f{sQqr;w7aI=4ifv#2xW})h32O@H7T%)`P46rA_RA`;>g8|Gy+j%P*0PQ9y!p ziHSIfl=!Pk=o5gc6b~d_XkY%++XmQdgYJAg?R@~_YjraqQFmbCppCx>LK$|xCUg6j zTo>RbQZoFT<9^67)&rfj0u2-jGLx~o$F!dXby%GqV;2uEcY>|IAqfl5hq`V%e%?lT z0D)4BXp~23OP8tSf6^c_{7HRBg^GZ^CU0N0xG`Q{&w2YapEUz-z=5;BVBbk z*IwXY4MakPNqK;Ot8aZNo3>a~Sh8k14LJkY&UlIJAUWf7ZyHYfXs*g)5$_y<20_g8 z`zt@6n_#q+u($BtG!P|=eTML6hLkL(I)9H?AW)_?S)Gm&wZ6z7ma@Z~wOsv%Qz)Wp zNv)9k-qBgjI;CBjuHCgr=8SICYAR!Jl-u=9cT1h0TK>5HbsjUBl|a&|-HgaC1a?Sn zkJmZp&c52eqaL@(los|z)!KWNgjm2eZ;<6+H$|kyHS6%b2GATVxWXYD@<{itB^+c+ zc_yZu$H25@-6fyw$T1-Cfu$mVXP16Fk%s-HLPuW^>ycSrO7~*)%|N+&Pt#ujJyUQ0 zi?qHDS*K&xn2tg%@}5=7&a=ITx)h06(z z*P|n;aOWR01yxmA!Vf0r<{MY>uKW&c)L=`uoG4Wjv{o5@BZb7|$I)3+8O*~%{OY%e z!j^Do=loB69(e5KLEq~4Ka@-T7FdAPDweSML%`8)dA>U>TzSArO>z4+arHA}GiV6^ zyJw={V`I%1(?|yXlX~h^z6H)YvEn@(zKfIqPaC&E>*FbHAT{Yj52MO$-0D`A;hH@D z(~`YXvP-4?`lMPioon^|?z&C+Z#h(#B#R-NX5yJ*&vI>plJvlGHGGC6d+Dl^=>Z3Y zzL**}K$=zwm~IpNA#<(|24VBg{A}mW=ZQTpa%#X0NRP>Qhjbu=y+-V2D`MU7u+}JQeC!_$9ek2_ zKNAKg#6-|Tbr7es6Z(3)vF~!OTHNF!C=NdCGUC6ZxoNOkNwz|j*L=Lp#P^Lii!{Op zS0KL*+C;sqc!#o6_&Ts1zNpc|OW;l%{#D` z1SG|znham~5BNMl<_!ex?G^Tb>?xI6URV?lxesdMUN$6$S7l=Cj8+04+^o&INc8Qr zEa`A7S_(xqk6DE+lxB_k+zbwYM_C->0e4$Jb%vRb(Y`jrVSW(?(q(#4Ag8a1wFsFp zKO+xF{wCz!^YIcN|GVbOG=RppD2MfSq`UW|+28pGw(ZYYxqvJu<7S@ZjpZ?>;`Q+@ zfnOEgl)#=$BgfkHI9<+xH&IfsR$^GCugsJaTgu}yZ+=ppXa9!j9s&m16q;($;A5FfI0@vI=&O1>QAhq(yY`S(!Z`1nA&(|1+?gTHlsR=`!* zl?$}-YBhU9VCTKlW;*Qc=Z76inAk-{RYLv~H{V>SZ&L_j}q`sl$v#C!BlC=bp*9r)8q!+o@yd z<5kiU!zR=^-UT&y2jYreK}HY4TaIy${X@q|1tlX}FVCniz3f%+>&oeFon#Qrnj=F` zPe;ivP8bXPc%IYlD0Z)ljMyhQyM5f3Olwuw9#)riJA0CZ7 z_H%%lj`aN3j3S_$_=fWI1c^WJ;y+C&R3~b5#)mAE8k9iQJ!T_l2cC`KZ4RAl;i7ym zE1#QH7mazoZdAzw-}w3>**varaMU!*-v{USm4kDP%4&Xg-02qV^7;gBuim3xgRXat zW_S90pYIrt*q!3ma=q+eO(xxlwiNZ0+K(AMJi(lTT{PeCb-U%|}p6A;fm zRvu`!#5o79@CTGK6T0Y(3g~pa_ej8&A>res3E=w#A}{!yE$J+jlEwft8Mk0?Gzhfd zwweW}r8Q1uAMjc>o5g2bHx_9a2ymaS3wkUV`pqD=hsj+BlUn=LY-C>S0Qf8hSUdg_ zooddXv&GIeFu#9_O8o2>Fdak%_q6X~PvpYY&D$fKhyN8>P(ct{Ph)6f$7^`1o$M>f z&9;AtiJXPA)H#6aQb$?-wY`p^So9w=#3t+5fW1x#IUZ4*W_wiKR6@YQ`(=Az7;j)L z>IOabWpsrFTe_y~YEaOomf2d+ZK0yQ%ryEdzoVa(zyEz4nY|V<2qYf5?d->STl=rx zB^c-9aOhp>RJU_JOtyl5&OuB6m1z=;gZNHS-%8AZvqL5Sbwn9fD+SS7Z=Ju__4XzA zg4p(Gq4#95BY04p@!bxkLjD!$WA93Lk+I3>3ha?@g2S7hoPGWsKsS*v;|(I2CU7~SCOpjq7#?ljR3*^?3H;i3GP?+WyB z%g~^NE-(++*eGx%VCczGX6KBt@P^%s25OcNJ8`Rr?J_KK<@?M4Kl(x!Vb`vV2l z2feqODg~cx6TICYAM8eRFFO-`a0KQhdGO&O*-`G9R@I!g*zK-mobf9_%}2spxfhPE zuUEKv`1e29cet)^z@78)W)W&>&mF^`tKF74?^{}TLj2QzgBZuZK?S@h*?c^UQ2>40 z`}#!e{;U6ieln6hTzRnl(RW@>YLyd>QvkirP2fX**v7pB*?KGfjfHe{pNGSPk(aPb zITM=C7ra*q@Jl&FmuU@r7HmN7Ut^%V-2nY&l2uQzuz78>!;tJnGudKi5P^_~8@I^( zlQ^-{M4rd|SGCS0oYu~bADDGFB_oL-9b>siUG{*fZ4Z{9{5fbrb$6M#A{bh^7KDll z`)Ll0_$&$eO1yntkcA)yP6wvI6$tI?;{@|owXvds#C+EI_byS3E?Whdn0~K!yDUaPUth$0Sw^X0*n>}i zqvP{e7FS2b4_csVR^RIaPJ)MV2|N-4_iiM~`3zqZ{zS}?cey`>A%3{hCfrVTfP@!} zdypjLw^&4vghB}AR5ZE5&#|PoE%IYtqWGIUU$=j#q3Io`B{rUS75|m3!G!JW?BV~n zZ@nivQK7Tm6j0a2izk*$yMz7&etv8k^1-@FpD7v%J}CM>6rme)pePC{b|YN=ZHq^} z8LKv?J)22A`%SHR;h-se1nuFzsduIQCyWA4m=qY?`mhy>RlQm}fs$L0_0f^iB~i3> zXG6gaHO;DamqzE)Wet^EbedE>-{PHA6(pWB^B-7_Q%lrN#HH5aD)OCxs9quAdEj@N zq#zj6s1kuVmuJLNZF3WeDL;2i^bZ`Bul8!)7+o^RfjVo!#&cyfLHJy_Gteob-#8V) zHMG8^!R3ta;h-0QBQ^4=i|sF3Ra4d6=D9`;FK+v`PlbST`O+{#|&gGPC+EBSC@*-Wk&;ogWZwD^Cu4=i9o z5!+holTspCg#=!a|sH5f;G zbu-oXUDLGhoa#zBBXR@nq3;#u z2W!9KGHv{Q$km`;9O^MU^##mpEjU#ZLNeYT*=e}!n&Yn)eE3R{3QhU|!i5vZALaNc zAaZ>?i<>p(6t;q?&i|r|$sGUDMW|ReqX1~v*%h`o$b5;_z|1p_k&5r-rtlo(Qp}tK zd~Eap+`-1uEZXX^(zGjawA??O&>CUDmC^|uw{y8}6=+93l8{2_enC|Y2i@i^hO=dv zfH~jDGHh}%kwCGA-Q0)^Mkhf~F=eYxMZi>8!QuAo!;7K3@_l!Yv7%s`7!FcUM|=@o zfQIfu2EL}MK^8qr-!8?;kIuV;6c7Iy!bGiWIK+7WoV3hC8&%CPPW=X=gL75hKR!yU zWWlT^oI%mi6xKfmMbTv}(4Y{s77yqqTs~=5#@ixyQ1e}drXnEfPv*k>rVS)(Px(-a z=AdG4Ol?hM3Fqy?8TH|ob$_G)(#lQ?%w44I#)5Mv)5d{H>^kwe1rw4`kkU(%SgfDP ziwiWtseaV>(Pg30k;L%$c1{Mb3N+F#-3YI|v4bN-*t*L!U|n*H$+VSLcxc%*uHwIswmzQk=Q&umZP1`aJ3UbZLH#X-ndq{f10vqq)xpNc0#Nmj%X*-et#X z->d9H#k2hTG_XAQg0<1t`C~H{r;WNSO000+2~&yQ(Dk<;L)**8+1%y8Ai9=9n0c1a$XcGo8#{)Tq5hy!~TA-F4jF7hG!nYfl9 zD%K8Mf29@PZFdq~#yaH4a#gErrK+8@`1kcH+5^G|VnSuB3z6JJ2~;v^*xtn6M+qDs*8|$Ye@3d>XCstMeH4V=|JVm)6V7Mzo}67Qc3PlI zSbdoY6^5Umq?9Sjh7{^eQQKm)-JX?(4dMcZBc(owUu8)IW~6$YZPY|mK{|+-6Xl=p z-J1K9Ek%EI=vOAM$OzLrFF8j{txgJnW0VTJfw9pjn_LW$0Vg+FW>i7DVEif#{V10I z75ceubG10|mSRpj3D)ThK*98cSV~DZQ!%9y+7?y+a~=v!%VYkb=k7B(Gm@7ec7Em6 z6}Ob?z@P>Xo^`TqG6Q5FG7`EZ3`}q~y7&z5X8N-V}^K`?u|G_na1{?0G+n!A@ zhDAc{C&7Doe;j&*@@&oD_kFBA29^B>E9=uipc^L;YZeFqvhI5XIUIbs$~R7jp1!*0 zHt_3fnTa2$f)gU#{}~KUZ1spR45lAFi@j97l^&gfA`iF@_In0w>VqY&@Js z>vDm>Mf*h#3&+AN+TFf6X-R~e&D|s=VDzZ493vuM%^X`dG0wu-?85Yqio;NOnH4|eNy1chg$caY6|K}`}q}aHpCm&_9%-Ho$kIMT)6f0So zsl)-X7xB1$*Z=J&?T8Hi4?k(tPna{qFbqX&rT_JJdGW!zB}t^CFDSp&7CG~p{?d~+ z>TYq()Q7F89rS5*xU(48rQXT6OGck|O1eDHH1XzOx2*xLzv@W68&8weTi9zGjaCR7K` z)C0yKyqscp`VS%OL^osFL#3K=Bu?Oe4h@u8^5xKm6?)AjwZD^V_f7=sJC6{+;FPN8 zNAsRPmNhLHn}>s`2BFhL(6>~ebvD(qX;#OU@b7CtQK(9vPt6lyz9pj>L0$XNq$oc1 zzIFmG-5uqqPTYM_i5g7#DMC1GOaE7%3U<%}!va1gIrR03b2S7< zYy~^s(!769lE^p|jNJl_XSf~x$qm+GEi~C%@~F7eisW(t78%X+-~Kq|o+})C;gJ!5 z7AV82|7DTzj=mCmbOI|QBnOp-QEI%kw{7E})D#O)CQLKLuuuHs*3zd*l3VzJ23$W} zVw2yN?WT$dILx)ktXOvoqVS?f`|mrCdwfC_RU+Ded$I28vydryyl~CyZ1*vl#{r zUHT_o&TO~1*UnDJMz8OtN_0 zKmZQ=vYwH{3t|C`PhOoTa#@As1?8KipQOz}%}nv~n((EBtY&2ySozQ0GJd@MX&JpN zmr55qcpQF7Gl-DPe?U%&8z2Ee{-4hv?mP7rkRL&toKek# zzeE>6j8my*)t|v9Pw93rukTBhv|OF}W~67*8P`FQ+Ntfzcg_ng?5vLOTF=8H4k=0fWRW5sscI_W z^kYjt=9C7ywHFK%PQN{Uc$|oUg^Ol4s4j6hXf*STZVlrP#VED@xZ0oS;Y;#g6`T$3 z`)&xx9>qPBVjjkqD}5K_{D{8rxCZ;H*MVQYx5RryIu#SfwOVg4hUd_wRj&PF0a` ztoGR5$Mckhej!_pM{o9dMUEL;qnsT5z&)^J5*DQ(0Zw-IrVM$*0@9q#mqcw5qQgk^ zX)9=8)C@^@K!E;V6`A}f+xU#=!Rhv7ssWQ#Ab!2(k&ls^lxChw-7Tg2YkhAbcoHo* zes8S1o%mlrKuc`80zJlEI!YwZEkv-P0YlJ4B|t$hoYe=YAFo_Gk*HeYw^*e6RwZu( zL5+r3Q#j^RrG=GP1pO$GQuJ7$AI3*Cl+eD;gfFxqXI9cf_0ryey;`ROw zmu|1I_P)){A`3vnmeD8W*hE#tQ`OylwJ<2Q>a4CkDB4z)IX9#)c1$>jTpDJ zB!cj3O@{0@vB?4y4Ra47GUUJ?es|?6FE=Ue;$uR@NW3Jf`DP-AgCi=;HCJs=+*kBj zM*TN8KkZvkAHjwEA26r#5`0S-0w_slb1Rgo{Q#qhUqonpv+4RKPk_dkkd`$-o$zmc z;MmNSVp}3;etQYjlTq{QlSk(;4%*8M+;9TIZW`Gyz)$q(Jn1+{Uw5pkk-~v z2*H@Z1^1}&Xe6w($t}lUDvL2Iy{zZsjM2XP-P5T~0+^!)w9Wr$>(Z+QwKTUyfx0tl zsz2J2-ndnF6Aqtu;T%Mw_qZ2l9nsv1hTdlOZ&X_{%CmL+`SH+DG!n=0B9eVhb)TVn z)B0#x2b-c8sfrW?BWzQr6$;8=ug}<@R`w%W!jB__S4+lK$ZHGFUjZDa9kqwyyy_nTP5Y6J|Qh=eE_uTDP3$P^0qlO9r59JAe!s& zGq||j!DRi@O-a{tdk0o2&E+PmSlN&2jInma!BtQ_;IPcvvvaQEMHk9wCL^EWq{!7? z<-*yxTBkG<+HIQ^biXM1H!NqH%7Ic06dPOxD;}$m%b9@4?z(wp=qEDDFzJaPj=wj0 zRi)f52_27s=cl_nP{4^XC1`hyOA5&H^^?Oi&VyCkfFXDr>TETkjN+ezd!> z=aw7rtj0>Mbys1^#6b3x_!~KbVehSy8wdSIh!i(?BoRg7wwM=QaZXijy@+GRb7YNE zQE>EA!hEDqg4hi$LE|ujf<%c>LJNL$b!du?pc3wztm$&m@{jIHH4(B$7Shhc1Pv3- z>9J%O^?nzU>i90T0Z*~eS;+9utADSq@(GH04eLLqNFe&1(9b0JP{`lRR~~Rjjb})G zWJYBy9;Iqx8CGXHbSu8uPmyHWu(kQ8U2aOn-#H+DWmBa# z>Xb&7p)HiBOAyN6j zv_*R_IC-HrJ|@Vy5Jj`4J+s#8Yk3MV3q%N8&G#_kxVb6Wx}SthKB;q83vfx))`Dhl zKR**X-T;oCfM>D_(Dl`o{R7Ciqxuv?I1MOy1r=t4q)$NTZon$V{^u#$1{=}e4L1sP zdSzxxeiVX=lygbS8(^j;Y!%}2XjKBVDFnRJR;5bM(@RDQiM`8&OZUPuZBQl5CwPfL zP}M~fG(=bUGPW)()$7DmL*Xd*6)Az*+@pTUS!Cb-O%QhWG9^fq4s&y}8t16#97Pr; zZpkfePOmiOIa12*FTi>!XvGpjlG7$(T%8*!ciprUg7S;9tb|)fC_Dw=Ij>A8!w#i2 z!PH5K7B>AmcfC6c_&BI`t$Hy>*s$fMiQ`fx<6J62nY}3{P}^qbzz|=j?OPpxHvHgQ zv-&m(F`ZVWtB39&#?ScgGe=kPWaHYFa8w>2y#PLPivejVUn~edd zHHR;a5V_XxsYE&OEHwCl)BAU|6eF;0wL8vzvAxLc+t_6%HUi-gfJ%5-FZC;kLj{{F z2IG|Ey!ACbEgBhN7aif+PmS)Qjt=YE-D#i-CDUuf>ZHTmsV#FxRZ@yTZVC+{Jh+I+U31n@x^(43&eg+y8yKGQFc&r~i2lVsM_`amfIK$HGf2wGki;lMpVdvK zy2a}KmPRfw+bEmsnKCDlfmspwqF6zxd#XWasyO$4kNNm(Y!IAyG#bh`*7ab*xVSS| zv>IKn=Cfbz5KZf7bT)oRM^Q-q5#<+)Mg|NuampqlHy#{?z4k9fU~akFOX%W7T@|T> z1O1y`mRopYNZc~nE6b8Tb}A^Fna(3!Axv@ZPAJ7HWscApM^&YT{U*2)pB>&S%Z(k9 z_!fIWe%??xP_|tvVDmQwLfT z8ZQ7fZwjm}fjTm5H9s134u8B{(|mZIEr`HGkeDYn1leOAz6*`LWJ}c-;gk1DM*z$; zUc%yK=?K8i@9oIkf3#qk6yHUOI;WDR$Z=}t;d8}QXE$ZS(f7Z&WMd3 z7&EXP*QU`7oQXWzQphB7T2G;6sUiEI%2lrGr!`Rj?fYh7lx*wpIreF&2;AgbTVEds zJUdS(g19Y=xEFR4GCx|Xp6<5g!CAvZQ>PJBf8 zru=G^g1`$__fr&%;x!gY%+mAq!zqr3nk?d5lnaE){gIK?y;i*!$+XLr4?jv~2_|?6 zD@oY;c0`rS*j}5KeL4+1e{CU3pk7AtfMuSu3E(eUk=5~J47+tTV6C(fu* zTjJ6P0OFN;Q*;iqg=xdh1CaxP8Z`uHQ7EAUp%eV28=o`~|2-OJ}9S6A5QeWY&O zCw;qr{@1gRqEQ7Zof{GtwYXVXq;>6f_n+$YwA?|LeNF!Fe1b`K`&q{|?A12X?CfM; ztCAYOicUFr^t_+r!t{1cZR(7wfA#yK9i*fEq(f2hNEwQ&vc)=m@nDgYC?0a5gUqs9 zHBTOrFp0Fi7{nTn%8nG7%O~Z4HeF})Gn=xja7qEes}3jHN3iG!vbR?+w%{>81eoEnB>e5U>`vW56{)_pahLaP|FL*=eOt7Z~C2V&&sLwGCP;vaPl6P~WFDnk zj77C{C(J4K%K6f{%nQCWXMsMAk{|&ZHoj#U^xpW8LiRi2Dotz=rEGDKsy#WkHmT{t zQKn>Rq!m02zrnr@8g_JgWozonp?RI>s7B+`4-D@@?Q{MJ*T3>asqR+a_4_#ups0u@ z=jC2DMBj-YurA$lG2ge8BNURSiJSz9OcEw1eiB;CMK7PPpefhR%APJd!TE~iHgj;( z+&tq&c*&;AO(>?6cUf?eMb^wBZuh__V#Nm;*epNzbU-1jj z=xdaE;C*w3NM3LUW8H{2g%WYhFWV}l)K1=j*(E(&@3rzmDsJTscq|- zDLYwJvZCOmw`wLOOpF9aTO@f?ESk|I*-M<8P)fYBWuZjICY1SPuVy2E?*#6LUM`uy z3g?{k+coKTGtSb><^~(&H`%YzMT33LOR~5~CNpO(KPQ)nAQHChVwHPh7?~Tjxz`JA zw!9gNAY=YQ6Y7hqp<r=A^~y zR1j1*=5QZ}@zpR5m(usuoLf{y>er}PYwez`{SEHofkQ&*5Q^DlkFW0t#&X(F<2bO+ zDS%ph^(HX?xP8g1aXot(hE5zGnHWDMCXwUbUXDC3$+-2FV!)k@8Dg(XQ8ARH4`F>t z-c)G2s0#t|3EK5|wrg;W=z5|?PsIe;g?50j+pR6FUI=ig!0?Xt`(q$4)w&0YTDR}k z;&P@m+PFx+aV{&4FY|anZpT*gsCIF4F*BeDkG4f;omBk*z(9hXGLODIrASTSK7#1a zzxGkfjTmuWbdYnE=BzLWAMq_Jtiky;+_@HPoTE{z*XK!r{pvG}?OEeoN#T7=sF8!+ z5ofDHaUl(KlU9_@ksE9XtJOF<32CC2I14+3vl{2H8{KfEn9F{aU#B?N)V)q_li$$BeXXj>eRs^`QQhch^L> z7lsjhOHs!WnrsL|6%lU->{kh@A&LKG?TXE~))Blu?#aXd_O<}+BP6^<@ekqsS=i=Ck1VLUv|&lLsKEV#!6J~ie2 zO@S~?n%~PWVxyU#1Ua~U-YzsKOXYR7ITd{rFc{AssLoYNh%cL6|711ZfJQPdnW@bghk?U{;LBBz$2IB~r>;6+a`Y zTlq?F1fMEr=l5g&zN?7mV!#%Xx2F(>zG49jJ9;aZAT5JapW)t4yjP{;9Wg$tRnj2y!-FJB8sh{!sH&Y1R_*b{GqkLgS zdKF-oIpG6B4Nau*>QK=?KHWT^eiHmNX#<(~?NG@;V7R&>V`x^+{c(F~+*Ba;6;bh9 zZkQ_ES0;J;(+wQhSNIVj6sfJRfRsxKa`(*>|6IMgq5<6-GSh}Qg(LA2gW{3I@fcQ# zOy;|La;JD(?w@Ur*Cmoku9bJ1n|nL4G*=eXRYK!$6LW{foTLR6&24ap0u2rd(*W-L zCZG+0y|G@LQqXZr%K|8Go8t5OYgPB&p;-p9;J-G_TC~av@3nFryh9o|MzWIHV@uyT zwI9vz=16mYD zoVJ8Ey1b{T$vGcO)$W7?VH#%52)kWAm~a2&Uvy}LO=2vUXV-z>4;#jY3^Trk>>Rzk zB7=s&nB0-%T&LpRm%B3iKQ6Nw@Jyj^F=`OML&JxL`Bl9NN80E__0n)F*Sy3lpW5ti zBhJ$)HyK&P|J*xiLMLTFXhEz?1NC)*HM+0gjqV>nAqTH40uh+s~Q=0mgoHDTY zhSdrC^}dX6gFJ;#&9FozkORMqo63mC@%_&N`-$~8%84Hcj-qMJUsjz*FEOV zhJSgxAbxb1`XMBtSld!IdB)ktbWeg!qU1Qe@29B>{S5e)$ahKODNpdb@NXz@3t zuMCSS@D*#mkSsi%(e7IjKP!$1W5n{=BKAyP`tUy(tkjLA{Q4k=>7E=Lw);1Q#_F3c z=re#MFKwb{GZ<`U4T$vL(HRJ6=qLfYKGf9!iI9IRk%eNig1N6f*f9Sk>FJ)U$(eg$ zTx9nt$Nc=XJ26Ixd?T7R7AwOEpKMukd1%*J!(+%HLSVdOvo9P|o@Pd&M*cUI&N{^2 z%9FFpBy^=zFlXi?68kIoyYZ*rf?X^l0uJ&O9Vud_>N$Ra;)N3eB~t`7?pVJ6S(=@{)at{VK`0MZ|kDACaPLQEMB`M#hMo41J z7uOQA^<}XnuIW>r$!-6$h{lwP$Nf#kDH$l zM<4Y#GoEz2|9B}|yh$8Cy5_xWqVlF}iHS>!H2j;I{m%3+ui-4Nv+io=clJ)axbX^@ zzgO}6*@Vn=JA|0rE=5~%Xf?9Qu0mM6n|C3f5gN;}x?-V)+!F4YFsf&aLWb^~< zGS0K z!UTD;tZJ2l_y_>-c&T&uAj=F}nmrhf zRWyutMp7@Qr5(iy1j*fvsyc!+m7sXVoy zA0^ORa7?MLxLC1+ws2p9M_Mpvzen4$oYz8bSrDZWO^3m6e?Il4N8^o7;rEz$l#Q%c zlmIz7Gau;y2ASBEq<{!zDOBc*y-xDfR@$XQWP+{Te zj^7W+-|qf`&?vVyH;t;w24l_b33f*Wr(JW;7v|0rE_OJWD@L=Jgo5 zq`~wnTHk+E9(5XIrj*m0+P*x;NZ}6q8MdMjjsQwt9I-%g2Y|L!pGT0aB=x@8{T;}s z%Nqo!Xlt7XUH|pTyfwNHaW?o9blh`!aYdc9wt=dc39z(|JGQs2Vtm+^2}zUOR78zu zK5cTHoSlC0lYZUQh@p|nHCzltiUL$B?SrKZ(Z6aTRfNix{l#yOc(kY4(6^bQ zE0{zZg#g;7btN~nPXQvKCeLx+tw*1;^4UQXj8SML1G5`n}i?Nv*He zJN>%-{8T9{XgH_gP(i}=fn(r&l1hpi%`*E(@Tr?VHN*6q;I)lAQE-SwmdN~S3+bYn zm{>zW@G>fH!(*|O@qgh^evgpz`$F6QxMqki4vhUy=e$?0oT;@tB0hNGBkWDnb%*XV z{X+eS{t-e1X zi@DspMhTs-t{NUE0Ug64uG2A~-F1d>K-cz+G_Wh-!ekO4KMdOaalE@4jpaYbe&BC` zvU0I_2ielj#xpb!aE0$Ek~HdcWGL{>eeK-Y}7wnH!^rUz>@ zW_FLZJr8{!sfitT09qxGZdM)-D7 z4aF7#eCCIK1z5Gc*Zl$68rNRn@BOpYcq^b>YL5OqB67R7G~RiGf0rl>@G#g;fn7}S zCPG__E7j0l>1$qlWScuS>_ zhtpGygX~(cT(Ghjnbghny)W4y>^EJ}PVNXzf|3z&Uf<-_+vQPN&tK@&(1lz;Qe25# zj*iBHtX|cU=y!~5vo>;n(=drp7qx|z%8vc6-Ssq0-7W!mV`0$jJHP{_bJyqd8W~hD z1OgO9c@x4LQ-a%31;SE;<7qY}f%jI}ns93v9(W&fC4jDv~uQLO?DXo7X+Hq1j{C3pSM7L?TyGackFH)PV94Z#7u$@EEQnT*Qgdr*b8- zhT$bq#RJOh(}_h|=38br?zo;~pCRyl)uCxArawYaz2q{y=KVZ!JJzVyk&thW?~IFv zz@kl6pw5*tzw5t>f=>hEf3Kb{nCicvlIRqJT@A(3q{-5$`+ky<#yXqJn1$%3+c262 zTrCD7zyquSVl9?asg}oWtk)ChhU+Nqr*UfE=&G7}=&;aK1S~jt5@WRM@EJIMfOD=M zC(y0d4uWcMV)IWxmzV!eUO+oQU^r;^&m-##=%GK)(E#Zsn>q>eW$`7wYMv>kCTV19 zPm;r5o18hvcC>}{^Z7eg<3wiZzL0Cf`a@sW?)LT3W8Z&EfBCyEVA`Zoxy79wuV_%JKYi?j(aqB+t-lIQMr8ooGQfM2#qT)1Za%^ zCcG=&v=F0@{iH$BSyUQ7S?e_GtdUb*Y6Vg#a0c_kqB*CTN7aqA(uziE;F$~!CWz&Fe&UzGvJXOZyCPC9*Kp=n z1*#XQKvaEj)`P@X^Hyf=H7g9=iNXlPH%^xy+F6cMQ6{tDPZY-dhBdpW$QCNWnbl@O ztapB&5Y2*ftU7$t_Iq70%yrCJ(Ij=~4{yLHHPhi93`EX>^y0U2Qq$ETDQ`R45IPh_ zR4pW#iKr}%qXxhG&g)&EJvG^{>R39`aa?3q-!prvBrT!txWpiiNdHbP^FTu!surkm z_q{R!ajX|L+)1BZ>L+)s!u`&`;+9@@8Lph13TriK!q6ED?r2ht6^z;wuf7vIMsA{l zY=IMyLJgYUe_#lF$?WkEF zFw0zh$tj3>-tO5*&v|{p>}a&P+lrH=b~S^OqI5J)0y?s$^18hR#RDBzIO`i7O1R{H z_b}cp?t!a94`J9FVQ<={zZ(`V;8+k+l&ddB_M5l2^9Cj0#}q3}k4;S4LXKr=SOf0Jm%1o}{220TNiE zysI8Le(bc}8(E*N+79wPMh!K)#@F^0In;Emo^uL3vdg+89h?qf8h9T~jAhsO{@wXr zvU2I0u2WVl!1-4%2u8O|vd6K+J*dh5D+Y()xy&%GUoGlpfC{l|Um!RIq~Yy(KL9Fn zYC!XwyW6{SUpd-dn$I?sJ`r9*FF5i+R>BMB5G`tTFHFV{_XdkoP_YL2@m~PRzpLX@ zfaA3>;J>wEr_38DE#9sp#V>KXiY`Y@Fb$zEGIM3bHYx4LeH9B~H@MCw}C1xDV z^#wEoRO|%~&0Nj=cVBC98&vG2x~5T~owcd)f_XGVz>(*i@aLA0twaDbR5q_#TT&nY zjiyR0Z06+ST3jR_Fmo7Nw#&jLaiobsa3d0k(@SA}wJ715DsuDvVXZ_s0`5xVokQAmn{12TQ-PliZ`!Xi8(-j6 z3)?9bF6#U`OGyzgf@IaHwC_r14m9y!cB#W@W zUGO|EJO*Bbp6B6S6l7isY(dL;K!rk*-E~%kDGBjOG`QvK(m0)N+{L+TCjT#axJ3Ei zdHjEWXQx>IX(RvN&sD`>$p`%a4T~z>1u?{jXoErXUG42_T=HR4G7a}3sw)Ou&zdAm^yeMc;B{{-P;%ETK0DOj%O-jfP+|Xs zVyPunX)8irC=Go76@Ht|JHu4sQ~3=dssei^o^H_fyF}M(N^ZVEbX8z~V`TNKW75pc zafc`t(=}q*wB?;u%Din8>r2uH>PlfgT->6NJ}=MyPK^4IcHz|1-o*WOw?uC)7Q6kq z_t!kY6}voiQOd(X&>O+nPIK=hjuM~vsToS*1Va^cmrNhDwo{jn| zWyYk>mu$dYE!)IuvsB#GQ_dcMQvMt5e&K6j{oho=S)=~n>+J01i_eF2ZL@_|6h~!{=OuT$_KnGE`C*>I-`(oW^Un`Gg9mKICHaffejP=k*-DgzmLV} zl40&IAOg(e|J`1m|L^YZeCy}`epf zG%**rR<7{x*Cg!s!3NU_E0rNO^EGw8N+pMHcD^bK3mduNTf7$L{~BvkWUn9s%;Ept zoo?R$zt{cd&;S2Mt}_0gaWj8 z@;LO+{to;}$7k_iKyOjKS+ADCZSA+zCm!Z4KZSaFI#f26(5%Db3QfXiX(Zi3-yEWP zw_cV`u~^jCWAx=q({#5$DC6VY6lhfV^a*XXAc3{@bHFZ9eW3of#HanVzTG-9rRJ+d zG;gj&u0`oTxs4c?go`!}hnxt}42LwlotYLqm;UeN@;`PuoA~d0x^kMCS=9GxP46}H zH-k;9$64o_L+DR-330B6DEn;rKW^(kJ;DFP3`;&t(pFprUlK-|LNrF|Lt^lcQ^9?yF%1@$?CZ~DKoNv-BD&n zkucs>dPbF1n3}Sb;d2#uNR(4XA@2BLLUlPm-KWA4YRQkywYF}gC=5g3IZnd^r z03H%az*v~gjc!ZzVczyV3!F8iHZ9hCik)+lnr1b;csf(n)@Ihn*f90guSNO)2y>r= zq>9-ux#s2n^g8+c-<_RaXT$&RhbI^3WnNo(e*bKKqCMy{K zVnJTrQXy$LOkmkgfO&kO;1)z?hEl)i))%rXHXXXs%n3c0itD99&VRGZa>=H3Ji=VI z%vsjWNPX56xG1y~r*^8KGQG`=T~d8gACX|(5~KErv8(zaikOg18VZSnprL=o(U4=G z6kl)pBp{MBb&%-R`=VA<$WaY`Ac+}do7jp-zm7SnA z?bGt}(mXaEW3|~o)Xj_a2CBob)gM`5Pu($NQ-3xi_3x*D)^RKu90``KcmguzKkKGg zF8J@eN*`oDLOum9RT*xTFC|NFQe89FkQXe%Hg+6oxEiX&8? zrP|g}-O(89sILCEy^T6xHhl!7;_-mRp`X~3(+6Ot)eq~_T*>VtCzHIc#pp$wIAOl= zC@tGSLX}o-;51g#v{WNmclwfzoI|A%1w}~!(GJ16K#_(t1yh#UWgvH;<#P+x) z*_l-Rq1Lu}@BfzrD)WluvD>pXw3q#I)FN_Ah19B%ZZiYX7Wr553Jfg+i9HJl)sv%* zj!$3e183kg_i)%kC!j_nL*>jFIGhs^F|gREPmre+Xk!Dnuj->)sL&5vuoLDE&-wAS zcYi7UZ;ePWro)hNvYtNX(ErYU-v8I#>vaw`^#4AtN9Y7gNw_}SORqY*8IcfO#xwxr zkFa-zheWh$<`gFpM-Z=w#E1j|8Y&qKqC#jmY=Gxq(rbbuEJx0L9Qw6KC?rE*CTJ_- zWI%6;uTPiy%XSO>8U_=@LikdFgd)OGKts~1wO*cGoJqz>?GZX=<1q`-@5g7zr(D!p zLn_3+97naM z|EWDfzhh2WEYO>muSBh7PWq~~s84WPzsK31wMR&8oi-2yQTz3e_peURY9s7jwQC-2 z*YdLN)0Q0m`zwo}CHw#Qm!tPTzk2)Y&#ke)mOf_h|K9FFH+TMHclV&P@&Dh;^#~o0 z6sHZ?uyzy#NGYF}0Z|MM&QKX}HU@?mM?!Lf$0(I*kS#@Qc;gKX0(40b_I-VVAh1TY zuUfU*qetk&iKZmTo-E$!d_+yu>-0L!&Ox*Ly!PnPBXp#`t~K?4(FdoyBOgbS@Q+)% z1XH^`q;eErf{^2Eytd?ow8vPelaVyzl-O4t?0%7Enh$z+2n`nS54OB9H_92P| z;XpXaZNvmYUMwUVn{X{>^6W|OPLm)(gr0g91#CimbUATkiCPn#k}E6?Bb2O7cu6-%jvm+t*fK5%|&pst4`}VN~j-9_F@%Q#CvjPWb4^A5ZP&$RB5>(%j2Tr@6PmiUQ0^|@AHwbqK{aJuC>)nZ6yFv zX*m?AK_3*4zS_cOtGBeM1fRh6;IU_p*DN6h2AL9403uOJ3>?u+HPA0a=JwHmlEAkE z$KeF&q6~g692+r?QpFe@p9+oIRGRQ(VE{LWzlz-V0tTM8U}&7q9gn#s5bn@Cjw6!1 zp@l!m?CPbid&FWXsIP~Z8gT(UIU*BK*^(Ke;r{|-HIQHw65<0L9^q>OB4lpZZFkhg zuFbp&a!oi6kmG>c60tW@tO8ExOg;dp^r5Ai)!8=R2uT$^1lFS0*$_t<`F2B(MT}S+ z_=?*iZ57w6%_5i{K^0@565YUCMu$~jj#Xc#-p0IP9-o2$T{e}v-GE*Z2lf^^B>}m{ zq14zU&<)`P;Xp7HM?PQ^4KpZw`s(QAyH~BTpA!8345P9jn48#f1yCQhjfb?R9Vc54xBOi#eQNLtpM)XFj%WXqii z`dPn@QilUF0|lC-=Sk#Y6A|&~fT{@;dP42?!~qM3&VJVjJ|>}TAtjI5nMlBM#{$@X zNl2)aF6Kdlj?7_Oa#4WgwpRjKx`6q*>xw$(DqLF{ovS3=Pr&Uv_Ce841tw?^b}ANv z4nxGD`mbcECPxMSX?XW)&J{%q|M({giyF7MWR+8W9id1r~QQ-zB^Ha)SPz zBwLgst6M8DhqF#CynX?5opNsu0xyBNO7#I!&XDQ}wUdQ}WD(PIx_|qY?*LOb6wU!m zN0Ic?-TJVwelc`kaHz%YF=jp;Opra>tnCs)Nv*=^(!B<<%zMIOj@%7kJ~pkjP$@#? zDn{jSDq9|@h*sHhKzD;#b8yMwmiK`~Fa>)0RWp`VVu2zh#Ik{FEB_h@S2`a>Of< z+oGI{Xi9{#l7(vezt4Vsp9q2GF#j?kfrS}5o8X(p3M*QofWkTapgol^KZS%2Wp5a| z?r2t~bF2I{g~XEB1Uq#XqQtE#)lVQvrLM%K-#6Al8?`N3avTLP_inFxR!;T1n$oku zK;5?}Ohd%DukVaV5GCq>miBM(M3>Ijei=?epK#)~KDWg7Uj|BUAU@6M?P1t;YyMb;-)Egz5 z#m$JAZDr@*ysB{_WH$(saFJw#k2en4TLN{;jxSw5+N3akbC`2OE@xp)&MdVG;y6uJ#+~P#=bdSE6NA4V9ltq4 zXC4lPwzJrjRLfiy^~%!H|7MqZK_Kj%WX*7PkR}18qM!=W-Yl^KSiZH3gJl5H?ku~} zfLR!fAQ%LwpNZk_+nHPmk{0+?-YH>5EBe1VJXG^X^|Ce{l(`VJp{D36>^sa8prov2 z-fDS-1K;)!k-61(q9UO|oss~%vDQrB%OY!5SkXA5;e6ihbTnYk51#MOXaSGOuh)cg z>XWGhF}qNEz0{@%iARLMd}3{XML*ONk#zFg>07kbhm%O!Z6`|A?K^wIMl1Igtd#R= zOI&;XZ3W3CRveiP26iKBqZsgMwQ0c)IyOUZWX5g)1@OkPkb0|MlH~Q7la>KV&)ifj z{Bh^+*h;cK5^PMerb{Eo=gxQO+lOlR5aW~pX420*4#?ju z0=}ogZ-R(lNHEqb4?rerWa7wcp(7=^aU$YCHqZ@00b^I%Qyc_fJk|17I59!_$aor@ zt?6I}$8x0g%wcVJ-e@S6&b#AumSc%0lO}kZ5~E zK(7-!j**O&{)~o1BBmLTujO@&`XdFQLGHfs>!rXU76udbp-B8e&Sz#C`e2lwusGwj zOWV-qK?n#TJ~$9UghShPXj^GmM`J&8rkV*P`8E74dGmjv5xY@CYUs(L0{NB>Iymj@ zZa`cn0B@jmwN6=&uLdds$H)t22+k}z%(ZL^r{aD;SH+=kENIN{C-$1G$HQW0|5M#9 z^ia3wN;XhE$z)YGyPZy_{zZ$toJ0ZkTyH1XNFx#bCdg3~9r3Y}n%xkZ!kb5FqR)#@ zU-}svS()0YNvLboVx`)T*6Vym1h|*20(C|_kBz~-WC3)ulvMX8XD6bCegli0Vr4kE z!4uUPQ!27###yK-i~FtiDLckmHTNL&ot>CKpQvga{pN&Pl|HAfeQnR${Y*?QUGemyWIRa-zw}mZC#_83jsuOlc3cyx&9|0QLjq z=pRS#-X^1*ilB}WB^wBVo}K6iy{9YIPprgdf0A6*$y3?HcygOaYTI_6ks4qAbz`p0vx)*#)6Yqn#)qB zzgiYMpJsY(8zZH4M0iMA%fx?{#sB{L{`~aKPrseNdB2_z;CcC9y z*CPbsdGs;{Liuw{eG<@+2#DDIMd^cD6@*tv64k4NRBPQ9dd>KizL%EjX&|3mlYm7) zr6dtjyI{4&M}75EJdPYbd~G6v6`pHvgqFtO-B+QAIYB3KI>vM3M@4P@qQf!jwEVkRV#HO4M#8pIGT-q^CkmLJ#>^;;b;7j-3*r z`aK>I?@CigNB|hqkSZ{hToNcQuLdDJP9tqDHNhYDHJCRiqA}w;)uc&#b`ry(^Rku* zb>N-fLjN#yQ88v~4Ubkgr|P>w>Js7_k5Mm{{;Ql)yHi5zt`>Uck}rlJG&eDe=pY~R7yN+ zwKrN((_1x9VUeb=Wr-MN83?(rw;SL#4+2YF2To|0QsL=<2q_xqgOPD`N2(vAL#d9o zGqhXLOE*^pNK*=+qeoEI_L-OQ4uWzh+Znpp=EP&%r{VAd+G=kr5_?HQYpwB_7ei?x z7z4zEr{vt~1_TR8kkC^>5DERJWK9zK?vSnO=0vnZ7tVmD4Q1Q0ihL-E7pc5PCHJ;l zwc45CiFpiIdhHE&1zzD3iD@7ZzGSg+4XSy3`ciCf15iH`iB3tx z(3^0;YPEj9@3By@fYg9zE1?He7H7UaHXg?zl@nA%*z{q`U=3xExz#i&li5~v?pD%{ zey3$imYU2i=Nlawq@b+TE)?Q^NS>wWYCi`W09j+|p9}R;ds|B|y6$6kgPT<{R8b4E znF)EMXMU&l7`fIv^x^c?+gC?tuP)vny?gbsbf)rq#FAjsWG~-jvkC6nPwr1r-bL)PmYZ--R8+w#J*3}w>W4d*?UsIBo5Wa z`Cu@k82z?P=3udvZKn@sYBQeddE_x&AqV|!_cWZ^F7$1qqJ}Pa%=Km8DQ`X-UbS7T zoil6WO*0}X5y;m$J#Gq%){{Pr<3Q3VAoe%`@p0>sz1Ot&ECOZlCr!DWi$dwE6|g9= zU{~0ki6J_-i=0iTlev*Dij11gDCWcbrqes=Tr0n0c1;AbX^<{^PgOH41oJ3ZvJ#h% z#%~=v8^8xTFkaVo%uHPZ_S~6E-kdjaPz}R+A2KPZV13iJFEzA`@gKyU#E3e^VGN;# zsTblfGyT>;XeQI+F3TXIRa&@Y_E%c>to&j2e40giWd7nGxh|FkK9f+UMU8CgSAcK| z*})+K(NI&79WkCV59?uF3)gU4oR5n7z4<%X9Tmjl>VogclsVPm-66R_WBe!M)@D?z z9hnlXX?3;{4Sg^vizF?ML*NPy5e?Oj#S%O(mz=oP zBSUJ{Le#+j?z<{IO zkmA*?Bs|1H;#7*YS82-Y+H08$%L$0I26xw!W1MgxQ|Po-_WT%rne?h+m7C3>co=%o&EQFxWK zJ%~fSt|m?aT}A-)@|!H`@8pP# zTj(|ALN?sCb(opa!VM*6;VlsJ5Q{kD5)}!9i=U)v>&fWQO|ePCNbp%^SA+ zj)ruM1Edu7@mPnmJ2UEpTs8ETHk;~2C6jgEHe^jtyccfbkuCptry zgb-d#IPn`vSi^`4$@rw7a77bAW)q5hA_*VUkQk$aMPAjqQshm*qK)?8EC?>Kca?+< zoA)y>6)?lPbQA?-Ap!B?*rlA>3iRO^ikr}XCsjaTKmDW&qP|i;-o#>QV-oRgLXJ2r8Z5+!-8g{< z3rRlzU@;m)qLN^U$96iMCXMIYlA0$1eQ=ejk6Vx2pIRJpn3frhAr#iz`!2N!<`(5V z6O*WIB?=m0M`%M!FDE6fJxMj`B;*OeKCIO~qZ7>WSRo(!j9wCL*i>Xr{n8r1=h|n~ zR2MS;ZDxPL3%TIy&x$^ST@-6^BNvQe*lZ?k!HG`?PgRJ=ARj`yfi1^SUPA|c?$?X{ zje0+UEEnPY86^S4Q^RCGqker_6m-3>2R%D&@-sTtwAboV5M+@_dM?%%nwk`Ud@34F zf}*yT+8o)>CNpyuSV>OyIkWj>IzRO@I!e;Dr-KTK-1hO+Gm(9o1)x}j$(Dpxz5OvF zI!H1tC-kBC#kz_=Iawao`aTy}N4AZ505b%k;Sfd%d$5r9;^2_!QM*@dy1g6KZED5n z#2*v2t|7!i#ACeyVaFr^EN)oX>?%_x_&c?lgTB+B8bbeKPQpo9g#L@NVAIdY#`PDU zy7Bze`h!6JDoYYU=*?%H*2Q#q-1OCec`W8600V~TJ2#i-Q8M z+N^7*T9{MAMepWJjTg-Ug_hNLOd6STJjD-mEFbtCd@4M4O`rr&=2hYD2e)8MvFwhX z4pZ+4<|7kNYNBpS!b+Vj&NxVyYP)bc91@N&`jcHksEPUz#X2^`ZP_LPW};rT?Ias1 z8m4ixsU^82ly$}U)e4gUQPa-hQh)}KWUKC(b22~*_>*03iEUjvQ-d~r=udgl2NT6&6mfQ2%7g64eRAC_VMVrxYCGT9b_-b_ zYyJ`HC!rX9Q}GN+4-Xh0Cs=u7!`IEoj0WMQX-P>it|_B7v3cEVl=I3e`?<{Bh&(v0 z<>tB+3gz6&(_xbIlw3u>{9bg+39bl4l0UOR-!R83_N&8RtM~azdj9fV66yI>J4?zW zt}wzKVG_EMPPgpwZ}9Gv^>5_XL<_sc`PxRgQi_E!Oevyxrx=?rm^cNfJH_0JFx&|S zSL}Pem^`ynt!Tf?J2ExP5K80JvU1KSqjdko`&lAu3u`IXO&IOMNd^Ri2KGjjT}U7GTp&5-pO*@fj1lV>1tZgi(YkPh=R2f{~K-z|;_;Uyib34O9ULv}J{XXy*7T zL!!s4M(*gLzMGu~CaOOe zziJ*qB#R#3-$Qm2qBlasI(qpLI*mhHXu#H7`Pr=mTZ8D}pBWnlBv+Eg>}621NTzCw z{n}J=%+^*(8~Od@7)6A@tXvX#Lx|D+ig926@BjV(2z#uXSxG@63;|;XIK7y@jW~kp zNsc*Ji7cMfvVi#lQYTF6kjV-u(Ll8gc|bY20gDRwHXPX@X*x|T$qz!|m+|$C2@Y*R zQ3pO9cudd@xrCIy1jlT-N3|%q={04bVj-P5#i462%RrsUU)HOfS-~9q7Lh!>^qK^dMv@X^w=8%d8M_jha$hMyvEHc`_Tg|0 z7XG0pD%rI!oGobl%n(23+jb+bq5805lUG&!sir{NTeXFC1n6%I@HV4rp z>?nNJ>B1)S77@$?O1N&*TE-`7x7Lto(Jxg3Z2IRRO1v+Zu_9jJp0(6CNxrjdyr+*2 zNov2AM@>iuGot=GBoYl`MHf;7r8buX%vee?HUXn%?s64o-O8_KkKw_9cv8oiT-zgB z*g&^v_S1)mNfJs5o8%TLBAqeJ=S^qukJ3rod);ntJ4;=IH{ zbw;NNvBXydK?;Wg+RP$&yiv{sH=taYP?!{BDF;QJs-Xfkon(yF-IkRRG~ywIkYo2( zKW(T5j-p_Knu8>ykmDN@i5LsQ)&3=+TqH!cZSbzyY3)2~o9ba@FE%~yw@X7hF0#QQ z6Fd&S0ifNmm~RDSH{$4$T7ym`WQ)ZIi4F3$WF+ z4Ceu{9Ws(J8ju?#h{r--)OK3v!y6mG_b543PZ6?hH+tkUmKm0PeEL#n6gVVs7WdXr zX@Bjh;V7;EZ350SrGwFtW%nBa3KYx4H zLT|F7NSY~Vjv6s#woD`&Lq0ParNSW^qA^nvv{GvupjHHfrl3Qp7m$9btMagTIMN;| zD8%a2#IxdhG8U+Gq%$X&+Gn?jac`t_1t+Ia=OaK6fNoWiNMr;jy@#kz##&&}*`kRc zAM?;k7IukErVSem099Blm7W}G)l(N29U}$gm1u!za8#@)lI$d?-s#SQElLfFd+G+! z1HnkE1jq>v1a!^pO#Lni{U-QI-Q_XSH6+ugjSXj%-vTs zqpc;Sa?jtMw^pWd^YekRl;n$fJRN8U;L_BPj%Vbw3&Q>0WCX|(i!$1v72W#mtV@E?v;u8}(YbRLg zh0p_7I2`Y_d7*_;5Y#xRxPlzBSl$pibWW{s>K~2pwef=i;<2!iA+TwEHn&`e$OW3GN#ctT?M^ph%l|1R;>-weD=!Y@0K4hwWlQ zqAa$`eP|{@BQ2yNN0j@`2y;0BN1W4su%_wF?|lInP90+9-L#-W683c zZgv67j9=^>RjRZC!W_-U2@_>_YHRo?(vkQxH4$=7mP(2qfLNDEGfp9zBr~^!{xqDM zjf`Kikt9=KQ+mwcA%G%dAAFtZiGLa2sI5nf9R>c}idG>*f&#W_Lox{SV?8S@)W;HA zk6dDO5w?5FnIS^xR+3QAYuyqJrA{)jChgSZO|YPubR7*3@|-i(lcJh|@8nl19K+X` z24L*55qRXrnu2KP1+j1OS~oT+c%h>*FEj+j+2t|@_GZyK?;4sSn2Ai}wR+!Yz(Q`Q z7^NYnj^E6K>OX1&sP828u80(w#G7ykS%aLcL|DP|AliG+ds+~g-l7{3433TcgZ;g2 zYv!TDQ8Q5T53;;ljaeUrtM!vd$?+_1X8L(DE0Ep+Z|uwt@^2|oNjCG%xhaA1{MikVwpVLf0UY{DE70lWdZ8J% z(E^%Wo2Iav&Y;z0V^7d}jXw8I69JKUo58pMl->HWU>~MpzjI{d7)K%(U7C-a(DFjT zE1Si+4_UTuJ$3;2G9lw>W;`InG2`UdGvi=7H;3H9oV&`lwG$=I`%L8mV;M{1Y=22p zK}*W?Ha+h5lesnFgXX+Ib4+G(mXd+;@4AY$=*jQLMqmBC!~>iP%N`5WNNq{g z3^mu7Y($|(Aw#q) zRkR0t-QJ%6tk>-A_j=7;Z?DsQ=J)oRJA;EuyhEO2Z}+lo=kI6E;z+awES}Swecl|Z z7urS%;-dYIdYlP1kga1jj$%pL3J%e(4);KuPa<-N>UHfqgLToNiay-o1pB{+!Q>Eu z09ZpHaClTs^b`uoI3AKi)cu*(kh353lAaPTz;vvG(+|;02&S(kpB)ES2s76SXfm<2_Q0iFqp1~`f?y7;uQIzW#IX~S;rE{2KY5Ly0Y4zCk zbfOTeqr4jT8KBQ03gaNCp>)3NN97$6+P8r zD7Z=4O-Q&H(a4hUli&3TLn9qEXTxJ~LIN$XEzo!bIGv{QQ{!W_d@v!gnxdQH)nhD; zGtT4}rgXgIL^Pn|CSEdNv5%5nuLLg|V4j2+rF2&{<8o>UvNhV5;el|VA@+SvM4I!h z{WwmwW~K0Tdk3vftJCTpc00Y@=SEN{RRf+r)!EojpQb@@x+th&gxWWo0%5F^;9NF0 zf1;s(q~Wqhjq7YR1xp7Y=~zfiyp}BoZtDE$5T;s9mwIwef64?Ra(e(Z6bkQ~^Sac3 z`pJpRG)(gyGcrfpJ;}nJm0x^jP66v@xN&*8nr4WcHuTY(-q=}|nMBVLHD?ru=yTH$ zxAaxj9JJOlZ(cnE)82I#2hj+39pazhO*R!+z3>RNS}pTean%k{4rAcWviCBpnQ0kj zrDQDnxdt#{LZeD`mZ>Onrp2AJE#8l?p~*tum9cc z?rrM7-p5s1Xh5;vBa;VNtGUHC)AO`KQ+>qLzn)Tqd&M$%=0<7T+m;!pRvB$b5bJjh zY5SiPM(|dL^XSyo4^BhD&pPlLC=M+z=XqXyQ5TRiY1rB|446yYIWc-QKJV}mUyP=Q&g;KmR z2Xm3*<4MjeWMl2$v&o#-z1D4;gig3%)8+@OXM64KmTMKvl#5MO@YD)-xRB#=y6Jvf ziQVUJ*g@ia17C@g5#^Hs4NaL3$dXByC3W^`o4lx2a2PDF?lERFU8Tu}cH5DmtZ=)tU%9W^2_FB_Zau@NZ6s<1CIW%&C z4m!?3pD&`TH>5{?I0d3hszT=qn$2-HSF~}WW_nb+-qGn9H_1`xuyNh9WLz$gKx#I& znK-RBrQ-&G=a)&XUdgQFf-8}#ByH086`XlVPY;^HOcY>%^@*+CR4&h)#nJl46_OJ9e?S8w+B6(0g|G=H zO6&8pTYm}`F44zK`M=xk>>uRL|LJuO4h}Z*|9xDaY6#UYE@&9VQq&L82T;;K8Ea!* z*Uv-smuD9zr@y{?b^gn%-_Gg{`xq2^^*r5-=|exQC!t?H?cXrKv~lW}-nNtQbN+$Jf>6))TD?}+ z0MRJ#u57q;>OjNd5X<}Rv^uldK=HU@muDrNn9L|XT|5_Io?tAH)8i}ZGo_%Q)2b&$%<4>8*^rgzk z$ZZ7Ri+H?c0#vquBwrXZeUa9lxX?o4MSz98;3Qz!zYsB4eg|=I0a08MkE08dmGsFn z$-lh#XFU$DLUt4Wv)*tU46qO{{`1fJh%!%jzh+=f8et^9|X zNjK{TKCGDw2MhLmU;jV5Z~+}tg=^hTr(`+JS+1ur8<~X^BX$FL=zb<7iPUF1o$R9H z;or$b;debhp4@9dbUwQGs0_$!5G@Mu5WftwpcRW$@ZXO$gI;ywQ<+1Q`YCI&U)h==2ryfQ6Wd3F4c#e`X@|ep&Zo@(T>xY0%70zeVc=F zqJ9^e2G-K+q`yN+_%#mlIFdy(Gfh!M!{NEc{%+a*EM}y@JNrCwN=b4n9Rxt;k!m`* z`K2+tF%D5{GaKurjTKO7jBj6JiBA{}L9%jIzaBzOI2y`#3`n1%Cn0Fe^~^ixs^{F~ zMNG&6z0HbL=H_b_$}{?JXmYRfKb_}rRATNMxHt6^82&pP&6xH9XJhr9f-gHkQUhLp z-#*GcV95tF=1xSIUj;PG?WF+?f-Fba={8We`>cVw&z`GI-F=qX_XE{Wft#v^-DY$v zJw+a#?`-F}!?b+_6!@40@i;HuYJ1i6X|voM2;`RH>6BwhrgL-~1n|kZ7coD_lGnqu zd*hGW+iEI1oeYP~5IrSCTnNG`5!qMlj(?3k3G>^{JmzGeMf)697>2t$10@pjCUUc_ zIIH2QW|-*IB-H9K6X#uyLxox-lm+Hv`EHqD4BOq^D9_xD0#9de6b9nNi;8>1QkS{g z1YWed5WXIk>j4{Xog!|?Lh^&K>-}o%`dM$ot~cy@!>;GEYsIMX_3(mS6Ao(hbcU@T zm~`t4gDsSW>*L^VXVaK@16&v{{#nPKq}M5TUxKR(iHkfFJsggIoKEqE-EY{v#psA| zkAyM~sa%=sKf5coZ)3O7diZ{~S5Z&zZW#ZD@oyM^HOBu?6!eCU!~e^F{#pO9+uZxu zdYZ6x)J6{p(&dGX`w=Lf?={f#=j#8hz1hOWnf9KbQ!EME3R&3P{k(U3i#QPx3k7k! z85^172bDQ|;?Xe%mf!iY><&A5`Quqd!^1}YKpXjEBY(`7Kk)T%o`=9V`0v%c@a01n z#Ez$Ezw;fL6v_exzHhcJnZ&_{v2PgrY{uT}R%Glu8^*q2>>I{DpRq$ZlqCcC7Gd?% zSvY#}0(Dk#2xMpW>oV`RSlFuF=rx=L{ev*@sd_mMC>F<-*?vlK_Y(604}oCTwie7n)?~X<`#!@J6cfviXewJf$g#DIeY`BLZKtr)M}!VX zuwcDTC%UcY_}zZ`Tf8SX`5^^lNI*Pt0K)+thDYMOJlt2UXW^!)YE;7cN7N^;#*v&% zdG>!1&Wi1g@omZbV;Yv+>&ypZr06z@(jw*drK+nY6>nJ$gxmS8Ps8 zs71K+nU~WDv?BAOuo^TZHwLoaEPX1Q0JdxpT?@)z3lzsX5u9D2`9i7 z1_5&@5GN7A1tCD+eF?=4O#ws3z&PmK-xUt?r+V=%9e?35S1U1~E0jFNt<{H862XY`OOm8X`uR z6TgInX+qV@lbZ`GE_m@c)&&;}$sd)(4T#TAVZX(HH1~7j<+Gf0nYU+li+;M<2F;z`%3^0>m1oPHYcfq1$z7>0)fFLo8{_2nCPpG{ zV?=X>uM0u{O)f+{zUQ!+yGz1Zhp=@@JVLKCMJd*5CY-|6>;g7X#U<1d(YLjRGpxYt zTfnm{-k4`Si{N|w`&eRmN#I3qf3?7Met z-<>Cb=L6tN1aRo<;J(ef8!Eh5wr32uxj=@CkhclY0Yw%6G`qlP<U%V49$gcThXlO?$kA^K@`vcC|Bgg8*je9zu=c~7E!QWmJrp@?=OM~r-Fyk8nU>I; z5%l2s_br@hqM0X7sq~e5vXQL8|UCUJwR*!HFtG8UxfoIdfvx^tT<4eNJ7mr0^ zfu(z;l+4Y-%~-gfW-Z)ZFVD$d2ff9Tgx=&wUZYN4iZAGiQx@-dy^H5d-<&15&>}wH zEMj31i);1rVXW2j&05W0t5*+Ut=_!@7aakR~nl`PqL)-FAZ&t=Wm{M9PziMWq{k&dUnB7jb+&HV-yhl_>HspAG@vj(%k_Kd}(^9ip zIoImK66M4at_^&j%qM?&@y~kJp#1;}4+NxPbDw6!n}WkB^RPm_fkjLAERdCz*vyaT z@7b9P{Cu;(4|{=MuDZa_Hw*kV7uY94($G#Szos6UzSys-FLw1(FKGn-Lte$XIkM|r zz@lh*2YdJRnwK-5k@20a=lv4apOZTO=W*sQ5#%Pv6-z|rl+fu1r)^d^S9F##JWfF^-RAz6>+LbtaRheGPiW{ zqu77m?WM7aMWCywjX#(Trh-wW-Fn(yuWFt*sw}UZ&rAmpBDf&85#{8<1bXRn=da>R z!b2j7Pyzuk^P*I+D2vzJeZ+*^Mok2#^sW5}ni6)dI-F6PWOzj`0wM(oJwA!#g-5v5 zbE3~;FBbu(1f0#33v&Gbm~cr4)WedXSE|3sG%l}%0+@;mJREW|#QDSYN_czC^sPWv zsHmRr>2K=78<8>oop50eo8Q;B71z-8+N7Z-=jB72CQL7BS(0%SU`fMa!mR2671F^; zmpN;Z+hxB=K$Ig7IFD_u?~HWdzQ7_q5xE2X{}mGs!P-uKobx!HkZ$Jgklf}dmdn5_ ze0g4*2~G-Z)%bersoPE(Nn9#V9pyP%B1=8KxWJiW!>$BSO^7O=dK!8`?31I?65MuK z-rFthHa3m}NlQYo>}rK&AI)1S=<-6x-OVk!J;GwtKxMkd_LA#dl5H&U5EfJXZwq5R z`?N3^b+7zogGG9@&VyKxbKOv>823G+mU}*^QXUtP-6b|dq#nUV{u9l3g;75m|D zH&ds+zBY8V;Kg;h;!|9%=n4SU9VXPN&n^ z-`$1({(tPf`FGn!wmVCE|Cb(&_dD=a#ww=V&v7=)t-QON}Mk)eD5)p_( zP=I8{_WRrC)CLqb0$faaxpi`45vb*&)=%BKbwB-nzwqzLus=BZufb7&G(0&u9u1HG zt3NzG8jk)8^>_7=t3twJ9Q{{+>#~X;_k{#Cwy&_w#JLyfy(QygHkAUPf@Ug*mC zq5LWvvYiwA#4cagt|&Uh0r4vfHg2osi_O0<3T;zfb;WxtsyrxHc6IRsyKj7Rt|4#` z&=~94Iz1gKXm$$P#!2G&@ynp3kIUntZWpPhK;3R0Ws#urW_AXc$^fiDQ@;m$4h`~z zWBo^HI2b%a!{g&es7M5&tL7~GJoG>4%qLOtHc3|fY(Aklax3((meF=KW2oBgXlE9X zSd5w{CxavXjCYjF^n#|ULfO)pp1kA5L@6c5XmI>^I5;{vJ~NdLA|SjT4x1o!Qw(4UeXx*LPv~Zd_w{>r4bVAoj4s$LKa5^FOayr6fqQ2r0!EA zjMR!KVlooA;*mUIHH$1ddR32ph_LI)ZG-uZRVR++Gsg;Af%3Uyfuk|MlaQvC7gF_I zu2In^o|rOMcsocv8k7wjY6%7E{2J7GHmJKjTknv-yjfdt_hEicgT;cB{knV7qVm4Ls(VsjcYohq-97nAJKmEgwY7JatG)E^oT8%c z(7DbiNxgYPW>rd|I(HN1URkcwdcoCOP*n83Q0080-oZqr*0JPd!V6sQ)^ihFqmrw3 zCQ<25VueGAdY2M)P9;_>jAh3Xd6CP`B}!w%aNAJoO*Kv%>fJWfI&P?S-B9DaVfA=k zv5pUASWR$PD92z*W?{$?QwVnhQJ3Gd@#c+W|SsL?c{3-P1? z%I^Ypu8zi++uZkca}#+9}S;>7~%7?)frRR0^1Kcu9h`7{`f% zQg4sC$G_U&aU>-$)ZroOp}onXiUg!qh1632VS>m{Kx)8e`@v^wRuX=DJ}blnNbT!F zYIT~hcuvGDOsD2fO{&8`J4M)5P;CfM?UO+@fN1v#(du+thNsp7A_6$n#>`15svH`y zWI_T3plx%mSE2z5Wagp7v_z^eBswYMKm1)1%rm zJ2Y7eZ1qT`+n-?+djDR_`o-a*f|>}7rPA*LV<~_}#q0qZ8INl8Ia^?(CrS4t?aRu29xCLUAsttSFy+aoF|jmPB}FxEE{YYv4krv-D`Xm1PuR z(yL{33MGcM;ZVOQajX?t)=Zu?gF~Urwbtm9yUaAw$hp?Sp?ocgNF(!F9d$A?*Tk=9p)@6p6dXAWtEHSN` zct;<;W{+SuUieGoUn~Qs8W5KVUz#Xi>Is2rXkS(kztmE{)On82QNR>QU`$Gs91l!B zeSr(6fej|l2UB2#DR9E%Sz+>wEd|b&0yj*a#3j!UV=%-Nd9y26Vwy_L7ju#?CSR(_ z7?bCW$+O0oyfIY`R>dAu#UG=cQv9gB;+|*3EIN_mk~#X}`^nAE?Eg)d1d~yQcb@y^ zB7v~~P|N?LUg&?jE}7!QkGpfaXlDG$ht>Q)rza=J#|8eM;qd_YfA$g-|4%zUmi#nj zP>&dEWX3&Wkct$zM-&1nagVIUEs}#%h0W&FBfk4ga2=L3Lb4wh(YgR`EEX%{`;T=S zl@86RZYnorD^!T1k_vHn)Ix<=?N?YD|5fdV&uY_WX1l4FkrJ-3y2X&pQCesK9v&Kw^W~Pzj>p+P{LnB+h+k98?oR0}m8QWj6W5<91Q{5=ig3-1q+@p1ppxfl_tN++JKNG2-f z6%LyeaC*0u6gXcEI@s5}v(hV?CTgr#7UJl}mnQ(Zp)vJ*KhIt^8azUS!Q)40@c0Rz zA_tEP^XN43tIu?J{$^e>t0AAfpNtBbs2QRD^heC0�d)%7lRH99cXZpf1969l`->y0_Y3Nhr~m1Au225c`S|vD@$2E?A^MK@zhbp#Gp5)o%JZ64R=@e3 zZSHlg9^bHnDC!(zBDmT5jcuM&+;>inAE77xP8AqaYqPFfW+- zgvB_wM2Mkn4uQw{At2s24*|-*sq?h1_m>(WB|~o4{Y7-m0I9WafOFd68~`+5ETDP3 zBS3QmfTl8205p|#2!KY*{;2>BOBNE!EXaaJ%d1ypadwM6pO zp{}_IDC3e_&msT)zyIk(l*Uu`;YS>~uE%cvDQ*|r|JQ#y9}ku3rU9Rf%%%}pD>G$; zy0uXEh`@^0qQwjrB#u0%>djuU+xKbZ0)E>$eye;50PfxZTtEZzDbd>sX!hyhTlsXm z&EfWwCiGT;-39{yZvoC<0y%Pt(#;*u0%rV z{N@V#K6S7VQRF4M)ccPmkAz>SMynDJ{lfBtZA}bx3>XM75MZFs83P^d1_KQN165`U z7zi-X{a_$*Yk6Z3^M&9Oal{A!1pvxd3s8z)Xuds!QnoDG1fe_ufKr+1X9Q3hpviDE zXfgz7@)<&tRklLy&azrrCbi1y3&lR5n*rzp(C4d$J{A6lH`31Z0sK^E`sIKhFfaOw zz)yo-M!^|vIQ@mf6Hv_nJ^_64mE)7L{qsinWCZx6GE=}Om30XCM9cmul`|U~8b8{x z`STYFfIvS3_y_RMSB-y$JF*~#OYeYx0RMa~_~(m7ETFQvAI6w!w6N_WU#K>}Ogys5 z>c1-jo0HP=+Qv!VU~~SOVgI-VHT5^bU{i@18fk4-lhgRg%nLFCfX=Bsu#fZ0rb%CO zTzpg2a$hWc&0ypEC&lTm)pe4sDPUS(+gWJKx zA;84!Il#nE3KRQ%^c`yHXD{alrqXow8$^$)0^8MJiWr+~642!i&}FSFM8K7w3Riwk z=nOC=V9GBPQ&w0uZ3ikx&7hL^nKO1-W--zc_JBGgZ#IuFYSu2mq~8K2{rvnN%FK4_ z1rdE~NcrJT+SP#2yJ=ekXYZ5D;;72ir*26Hw?ona4KN@olAXx z82@u6ikwp4C6SnlTEgf-*+Y8azGBT8{^M#KO|a_)vkD|AAao8tdkc^Et5LrR7ye$s zX5ER1BjIwP97b^D%qyP9IFd738CR8Pw6m%6IcD<-#gWUZOltW4Oc0uWfqi8~r+D0^ zPgGoMt?3kc_hcgOj4?Y`xS|Qx7WhJ~V5E$BwZV$ZzM88K0k)V)Huw`x2>c_((+5)f)PA$N;`rmx&|^dKjRaGaG{tc2F}T}XF`!ifGxoLC6_J&n+UMg)0* zT(>+L@cN(AyNYgvTyHuh5ku36-b=GQLZMGEBk0y6ccNOJ{do5JhZjHn{DZx4c}4PP zWj<}y5aO~aagirG*jvr7Ez7KwJRju{MI@w*=en$~KP*NeufmujM@&{pL@MgD{5fxR zB$qmdKuQWB5r314bvv}*^+?b=6w-7mvEBklH!i&ks-b127NFoJeih1Yg3?v1fG~2@ zD}eY{lT9;cP8K+mW_Y4s#7keTAW=%5Us#AbFgV6)_6@__uYs#uP#Yf$5OpQ);oWs={~5Fh-J4k|Lr#T*T27i^@o0boA$!bAv_}?q||0D8y z;zeXZf|%Jpb#AsTV>SPOe>55v`2PndN5`kY|G$sml8^*02^^2GvDHO&t(HiSn#;IZ=D;2d>|t8wOah#mBGlXFT#xM&yYfS@i^E-i!q7go2Weq?l!ghL zhY@{OC~E5YWZdi7Hvg{}w^z=*Z=2NvnM3A~Ib^=e%w2CrSUg5-j)%u5WAbE5219pv zbUbm%$;r{=$#g_UqoL!D@R9T6h@3nbkjMD(U@+|CDLy(G4alV5KN<{$x-|kxU zGk5Wf2604vpG2>4fM@I0b$up<&!~#^2|njV?6{kJfl5~vn!or2a-K)@-EtE_b;7cw z;>{-z^}?AGH}N;?8D`?L$m!c%ie0%~YcF_Zazpg;`%ff%anrofCloU=K7_w|q2BEB z6QUb3+-_E>%fS%7GRm&MV(rPqNK}w@*V72kbNxBQOjQtY=d%aTbhD|Wn6e=0Cz&*& z8?tquZSJz4y2ozPVJrLVC+Myl_15an`YHPA#$C0(r+$)-dglc$y<;ynSr^J_4S@?U zcOmnA?|2}Xp<@4EyAA0*<8Rhi2{i2g`=>`o`ThTJ*zX_1{(m0PMvT@OrMim{>wz7eB;Y2tj%EH%M;29YgAcnROT6j@JBjR=)8WGC6m71*>&-m0M zCL4r8ajnSroaGq3oW7>UxxyR6*=ljYsu4*d73}GZRY}3TQAhDRw;yFPsjn@JX3O-hu=%_H{A(>-b+0K>BxKvr7k%a+44Mnirv3s9|sP9tq?B`0vZdGI3{dwEoq1= zotVlebIRh^B)+53P4*EJ?2Lt+FQ=-N#y(&jM6?1uGHd&X`^`J|#& z^qv^>YWzpuEgumPnRzUZmOZf@shBxL)8y;R=rgj?e&GNHGcrbJ|GGdg{&9g`p1nd5 z_0>G7nj2x6LB*$V;ca}!Y`nmK&_E1fa>IJO5?S0!7>Q1y=2){S$0*HU0#?y zj~9MbpU&oF-V$mF9}CN|(w+<3`Lh71P^y^*7c!0wYM++M5G*d<#K1=R4&n0tWUu7x(L8=^EN4Bva z&as^@0{cF_Bkp-br=BnTezGxo@ZE!T@=%uRMjm7Iz@^R&iS#?2pRUj>!wHF7@7%^p=AyVd7)8G4wlaQ84Y6cF3xo-HT_0OaK>KK;0>j5{^1W1O~NrcIX)g8>1V$% z5@nC3`8a$)yrwQe;)0f(TST8^%$o&#xo$IL_+-+Rvx!jfGvX76e_PMuzc0j38p)~S z-~gRXr(VDZFC%g61+z`dO?#J~p2aBY$xOsd&5Q3@Se2Iwa7le4Bk#CqzAVID9Jt*$ zEuv>-f2&45Kxf&YSF03uIfmnKaZb2Ee1bVE&bz$wgq(aDj65oOijNt4Efz^Ck)9`@ zr*(kBViYmYTFsv)SG_0+x;PZzN_>d13c8aX7x1~qqC{-b|DL!r5-TZD>Wv3E)R(Pa z3-D2Pe+2O@KGtci12Z*;h-xI*R`#3^5QjvuH|0_iUN>Ga;~<7GzTj#EBSc@|(0CNl z;5RzS9=Mn+Xuyb(OhR>Gkdi(lfh!#Wq)+BF^8VnMzz?Y_(5i!dzTejoghY&y>`AU* zamP*g1J9letd@x^2eEWMsW1Kx^8a<`I1ynH}s4Y{jEEdxU z&-j<+YW^1cp4*~oZ+WM3%1x?Qc3Ckvj zHLK|MDh==W2L;T;X9VqALhcmP9-GDFa3%erW6R{J3<{kbpnq||oQREOL>#(UkiZoP zKBh=v#Z=ZbwcN^47e)zeJNrHsLfT5qgkS^#faFy6|Eqw=&2H6TxV(kUyoPvN1Y2dgZC_Q!`pFdLi~` z_{tMq;yj|a@~YjIDSh&=WWyUHtP8GcVAMBd>==qt71 zrEoK%RJK{48rXv)`w8!GDnOdi%$EuYbOLabd^r;x;;_2L1nFG#V82f1~4m zAM}6w2o<|EaWj{BypyFnTyYWC7i$sI=;$9QU_oc+FGW-mi6N+U2n?kvHE}JGrxImp zg$14bMx2=0*5#b269YF*!c*DS%YaXIkR^BT{yowPNY#cmmkWLT*fT$PDZW%Ror{Pk z)l&?GK1a~xU5xm=6w#PYk|~n5qWlxpH=j8ieWN+SE{W_cKUV-iiKF?pp9U;hWCcu? z<}GF~X2hTJf+kDe@q~6cyZWuQfj*Iojd;&xz`3i?zQ>mg-8!}f$IR`*hV%c`ocJ7_ z+Hu%?6x1;P4^M}uh57&Js1NqvdkOE~_r627-eN3N!&A>E@iHV&7ra`WIT@qxdLKVp zJeKw1U8oO2Or^8vF1DrObv3rBo3=Wrn?)4x)s|)jV(+ZyEFLoj)E_vc4;*wd5`TJ& zi)1?W-l0x6%Z$Tg{cpoDS0Hkg5b#l}3_OM&DxV3@UtjRJe5T?k_t3UpPEkbUoqLgq(qAil ztrYNadb%X^x#uuQ#d+MXEm+bH-E_V zx~`@3(QI|%!y_VuI3Ur@^Uw<%KXDs@$zfgA;=76!T_{Am$mz7oq_#HC_;ye_+AxNa zxyx71bX*dMP1PdtW3QVS1TTNka)7b*&#sBFrmt2En|ogHOVluq8oBw)hD%_G#o zVaTdP$+Fm57-QwzSeh)4ZN*8?Z(8<SgY$tWf z7d-B^1y-N$wv?bc@vcSMRGRGtsW`%}kVTp& zRga#kEOsDP>nfq{*2Nje^Bb?qMjks7r(hY-vOcO4Z!eN*jnG;YGtsspXhYU^WvkfR z7b?vwT9+!LZ(Xcm#8I`x)$~C#+4drsCLs0b@p)_f-0iqv4PvJ))OP{!7uP_&fU zuwqO3Y%N%&iO*^{zFNY{3|ou2ffnt{4cqJgCZ0GGUZDnTvi}&K7W_XAPfkW~|8pni!Fog4;y##ZD=h(h?J18C`n-@Q|0}wtK)vS=+ zlg$o*)3?vlev*#ej~{c(!Tu+aeJnwpc56_laV_aoEGBiS%%!VSy>@kqjcZA&&9ua5Bnd4WrU7#*3@F<&l1A}-D{ealG5xZwEuC!@A^RNvWMW5do-KSX6)h`KJE zUSfE?Sud1(p=Un!7MFT%6nBe^nn7uvs!oi~&x4<D65dHwltoV?vL^|t8vlW=rG3fOop;`O>@;(S>zZEQ8LWQ-7U=bM=3(-@}Ugr zwZWWqZ$38_$Lt2?rNk8}vuLHn^7dj=S{|#EG`%6&lv+j1SDVqI#XeNJcb6utxNm0^ zUYoE`9zy8_^K^7}v*jv3ySJZkWFWz9)~q_$^Pitzy*_*O!XVNSPC34h=wh6s6%+P{ zYF|)#evae$IGtw1yii-z#rfH@Pf(t`?^LQ#gm5&ILwkL*1~Z#3>UOhKPgfW&@S?a3 zYW8ZjPgqqc10 zuPDhlwQiYh;c$9Y-h~6`mIa7&=2h8>!{v6F)tn)($i3#^c(dZxDyelLRcF7e%Bec& zZCzAT)@`#g&uUlXXdcI|m2vUZwYiMqQES`cawn=Qi^(0Mw##0hmafQGACb1rP@ROX z%uyYGw#?Ez)h$(%FJ^|lsFOpeID1cxYDuZ zs**})ll7=7yAx2zFMaXU*fr#0tK-5_mQARi=N?k`>7O2I{C!w_XOy1B5-9!9C{`y* zJzHs)d4|N|ot^p9E2WbPA&p{Y zZh6wp)w$^(P6pq*B>`dVJfag~?m6ah9R5J!{5B*M`;Xq7U_YM!k$+Xr8zC?7Vh{Vz ziI0~T#G!%9#^~6H5PJ(klepqtBp9>TmF$y7Y5f3@DG z=twOqtK?#>qQFw>w(277ILJ^GX;(rO#iX%U>W8E!7Q-jfdz4;-^1UE=*EM`7(TER% z4=5;RXig_acGaq~T#N?ca=wacWHaJ4i)Tcxmd=b&E8KmbRzTQw*zo)>bSLfI05{ry z^hZViA4jJraQ?TCP}a;Bor#s~K$OG4ywS&bIUyYL-QzLeKF{MSPijitC|y*B1xt?m z9s=KV*mVA^dZ_JWP= zf{O>}&4=eJa%i;c2qW*-^U2Kgs;r$-DzobcQmgl%h3rMAyS1xGW5Qa&3&;LiX8!Qy z`Lq0k^Ow)F2ibwQ_^v4N$rwF&m9Y4YaZLR{tgS!s-B>Q3h;tS!#rq@-eImW>D=YTO zETv~-<#|*s%fi>+#B+s%V&#TCqBKS`o{^oNM=!i!j0WkBe}Uggr^3gEm*Kq@8WHUN z9QaFKN#A=uky<5Py|9=3%)+MYKMT99eSkFW|BghhRqV|4 z;Ruy4b7&Q&*V`W{)BNy!*pHrbl=nH7aO)) z|CRUZ1{lz2|8Z2b|2RH9hW+1OLTjhttAK*W`LK(-FQ*vwO}Oqk;fmz7prVcSXCdEu zW6+{`2+ZxDGHf^ht5jRg|0kp3{D0CP0RPusLM`~Wb=%e$fMgxUmhAq5jjOmayppkC zCwY%M$(4J&w3&Sy07%T4G1n7JNv=q-ts-6jiw#`B+};`z_X(FpuM?Il#4|5V;$+tJZa>C9ga z>RfmF+E#|oai5|r2h0<9q4Vl=8r)oqCo>Y4lj2a2b!>ma-p;phDv0#?&*UG%p(~vG zv=6(Q|5sT6G}?b1745%HPfua~-%GfM`M>JSAC`gpSq3(<9LSjrY}069J8<7&t@+>P zUD}%kG$QMi(J=q_j|WBmzu^$QOpHAcs-V0`rP(*wjd$&Y*KQNx-z_kugKxSe}K@Y=-Oucu+ zm9*afeQ2Yf1AmEVAX0LXP)H)=djYX6`}xJyMNA`N9iV4)v7iC^=d%msdJ(hinHTrO z|E2hrJ^3T*iT~>d^I4DoPyb}MK`+Z_f}NWr6x_$m`p#x|q4k|T!8g`-cDx9!@BY_1 zK>x&%M-zr#K7YY1zTfB5mSuY`!95u-qQ8BK6UAENKPAs!uZ-2>f5hJw#{Xah_y6`2 z4p6zB(T~JmAc0IQ>pS$4LkQnz;yTdVis*Pog80{m^EeLKxYwI`@jRIbb#R77D!S95 zZCM8gI(vQJNELfYktbhdB;#_XD3br8>&3FG9$b$t>-ze7g4x{qlK^08E-dCiwn7Y^ zKnU{8^oC?vAJ93D@Pfo7LM_4v^qep!^2C+rHVIk=p070GZxh-2Fz&7s){OpX^8H|a zK-WdDLf0$nSEh%Pp)>+9bWyyUo9o@rEcmusW0wbU`5trE9F z>KM5=4T3NhgK+cP`8oPZ+~r5&#Bbf8(^!(L9dg=ghX8K)SLMZ!UuFcXe+do zx=II7m%|5iK?H&QbsQz+n*YU<}Z=R2=T2x!cIgPGixQJF0;{#|8xH0&C6FW zUSFR5WJoP8jkc0pNNrl*HKmfHQP}AzO8%^;Y!g19>&N}a{dU!zliDz~s7?I=nI6z3 z<2`m=;_|=wN2ts;kox_{wfePbZrWVe&qQpcWf46j41FW|CH;2pT597? zcw@Q`=wEXZ&-rkSP(Vc&PZ$x^%=6Lgq%F%grCzgN7Y%Dy&wK@`<>(dWyGUB?t}C#X zS`P2lQX8tm7Ipn{Do#WnA*EdWA5qtD{(kl>I}X@~)Rh)+HRJeZGObfDABcgI~5e@t$ z@}_ETNCohi>hQKPIo3#BU)@;`38`zEIU+9*bvy;3!Yl%(DXRNax(zc3P2&LM&hX4gV*Aa#lG zWku2tNF0sPDLPlFkVqyVD`Va)Xa!8Gqz*j4r22CxSIsK_qMOLOsnjXXX{G0@(2?s5P|9b-H@Sk3rR{98!5+O0vC_59 zU9IS1s;;MDO7pZ^OTFfbUdhW;y8|<&&RCq*m2$UID@vyo{o0gTlSFMR^OjO)q)~0^ z+90C3*Sa<+sj4%`OKtF6Z7KFvQl}Q?^*U8U>WoBd>n^gD)O8$M*OgM&vUIKg5nD-} zvU0Vm>x}5@e(QSW{85s6NAt(lQXBkYEk>hBT6UkKaV1sRby4cojA(0p$5v8jv}bMR zYlBMdUgzrqp<2~$*Oa=z)V77_+euB`)zB|5wV;37$#C3C>S`Xi>l&$Rc;+_wjGand zs`r&rZ&B~tNnOEEcU>=a76094f^RGJ8mo-Ze0&D0j6ty1VwEvj`|fj3DDbx`Dy*Ixlq-Rpc6CiXX&fd zifE2wyh*CvBB7 z(+aaWO?;P&s*lQ73WBc1tb`ZF#tvatp?JQomD+Gda9zm#NB#=Q9CU4OAJ*?7>9=VQ z_gcTl6Ayt3Mw{jvG}-$}{;-)Pte|9^W41#hqp zjj>bI)~2Fh(7Y6grhrC>tZeOKZGU{YqS=kcWxH9V7L$I-YS^SLSVW z%C5B0sZOXa|8(UwVcny{gw4Zy{xQ*p9bBZ2FLuvUelP2 zkvByNXDSA8gNMDa@LG*Rf|Al?0lQn2F?X>S*&_j6>Wr1>23Qn^@S zdlwLC#l*z6R(BAFdg|+QwCkc58TMP)aj3ZDHP&g>Rc9v>| zap?_0#)sX_J!hU@TJ`w$#C1a!XK&Kark3`+{$zCWMCAB#ijshl_>t6Xh}4?=N_C}Q z-u#rBqRO73zCwET8~1v8y6e7wpEktDk6jVje*d1Y;~~Qi`S{Uhw~np0WVTNo?2q5S zm+`n@!u!Ypx=1(m3+f64#HRzZkMj9a_6-R(h>K_vGrb+un?9*K$OPvjkUc^)o|EX# zV+8qx4;?Q?LE`&@t&_f6{u23P>Cj8m50jVlq*=c?lk&Qn(&!FHF2cbwvrtR>fVl1S z@|-6h3{UO8-M0raTa{dlX*oS<2~{4d^37f(<`G^S7D5RlKk@40C1ezwg@GuVP0kBq z7^!}6>tXcc<>mQ>h13KuhUWo#P6;!=kPG8QL`D*pylRfg1pB{5(zK^)LE2QEQk>!R z>YZ|mF6UY7lHU{lJ|xk?V@%FEfh+RFXzJscDo4$7Dr@7eSux+3-jG1ABSN9&iTDeA zLy-2{LtK5?OE-lmCSfG+U*}mwToS|{_St%$?RrcoKQAtRGA;qkiE|@M@&fUcT-$d|`| z?nSEK3r17VCt_zIdXX^qM|{M_0+WPm5p9sqlmDOilswHrOlPoB6%TcnKxva z`|aRoW)#uC-)P03v;fU>+P(iMBT3^|CKorpfbk)wAp=!YhHeyN8xmM3x9W);k-6@T z(c52n83J@Ee%SOLeY%CpCPZ0=qMMkF(XH4xbyd7U_MYPm9Jpijwo^#l5d}IlT=Lpr zFIShm@S=#XlwIGOL^xV@Ckb0l#J)<2FtA4hdtg~&zG+xy;(0`q*<39*g{4NV7A8W{ z;DcwKt~G*N1EE&Ur=nFfplBHWfP*89_=M)`Y2Dm#?!FE6*VHtjeqGgq;$7$0``)WS z*nQYw|DSn*yqXg~BoVXYaFr^cVgG-0GA#Q4IX(gX-+saYdZTo5!sDGwrd~i$NA7X1 z6okIgt}{Bq=VQV9wcI6XePHG8C`}edUJy@F=g+Ly*(d`|3Ss_j9iV@CflKcgx{$8P z_{a3Vv-HyxNArmuyKc5Mz+MmwU?B9Da&3KN@hB6*(X4!I?igc=?r3!5Q!I?36u@WB zoVbav@6%9@$+H}-?|NKJvx3FjE`QrKa+menhY~_VGgfK&12UZwhwmF- z(+ed4f2>bw$Efq-_XPW$d~MZ;p*^za+PJ{;`SA9fLa-p}e9pRZy z9?nYXm5#gGji;ePFqJu}8sY=`J)yCPs?|?YAua2>jYN7!ne~o8U7_()^iXZCT-3SL zP6N~ z)$ONS-bkmk2!CewpP9{Wo$;UD4tnSXj-R;Yy42oc534%OLq084d{2QIr!X9zNfch! zr(`|LVvF-}3ibZXx@l!+O#@2H>%U3zF4xY<#)P|3h3^mlZH4~}TYT0PZaqo|gH8CqU*!K7^+zY*|93B; zdTm5@$0l8VK(u%g&Sa@Y#y3-tDZ^sDmV*Ssc&KX%Z@BC(|iXrQ%Lzr6X0e_W@h z)TkZfi{5r-B<}obxu=lp^4$MvInq{)K;t#pFqrIwt(_^76c5*IFsJu07SaD!J#fzwR5KK=`y_r})3dCU70%!jXqTk0 zxfAaiS#u{SZN)VYbyiAiMr;ta3cB&&223K;bXW1c7qR$XUOfL%CQ%Xd?qFZ9zdJyg zn?(an$)Q(e&LK>7gatvqoZ16GTg(lmN-EMfppPHNwb5cc%jC^W4$=p_UIid5rB_kS z)M}e;n@Tq`Ba)AfqZ!Le>Y{FU`nwxE{nNuAf4;c9dU<}Bdnfd#-6Zm#{^{ZQo1g#B zzps9I^HV;G+C+D;>qb1*&u=dC5d;wI^2y|B8S&{4G!5j0y{@*_%_?8Xbbw<6BQ;x*|{ z%z}meU6%v-r@dPo^`a!`DHj60{2`_q&B}1EClYQfVgbgd()jY#(;kVP9%H^hWwPTM zf7+qAJ(Ev&LzWH6x;Lacse@Y-+I4W3N6Y0<^WB9LpqKmQd z5J`XFO65~}&Ua&D;fNq%w%k~kO5&rqSj-qz&1lM0zMO-eEy~%fI;#;!thSk9!RfB~ zNn+3cng)C$oLg_wmqHX7Yp-Od|iHZS0X_)RN$ z7qPiHcR0q64@OTQ&YYso1;LSE3p8wX#bOvPg$6|3j#0LkvfFh^x$sIFqd&cW_TtUu z)%P!ddU5so?A42p<%q({=1=ec@#5brqsggGVU8!tnGrbt{?M4hsf=mTPWO6hWBJSW&i-2e>^jdD_X*D0p(kQu0F>bvTT)Mxsp~$ zi7VwSb+>0UT=F51#+LUm{!}c9XKXyYJbUxQi%YZjI$=ekQVR6pL-`Ax`GUIW=;&x= z{94g<^(n~<)h+~2pH<6UOj3;BBz1lC`ufJD8ZWi3tn;(RS}!N8NYz;LZBkU?r2Gi1 z1RzDlb9UQ8RsfCU4#T`~qpyDRJVuZrs zN>d5zV&UA$R(rwqv6NX&p(yHpj-Xmph(20V_g+vQfWt}EuEK84f8;=|w*G3g{~8U7 z{$ED@!2tN5_Yx{LvGOvS*ex1cY`Jvo+5D?yzS?vYlDClkJdM9fAGK!#tDRTsCpWJu zNxTc|s)}m3C*!ILNp@j_)X;37+O+C^zN)ohmHBWJ%c|6VNkPp$Sy&CKO`yom{X{z;S zf6dy)-Qd5xTl*pqY`}lR(NV$vdo<{ug8lbiLSbgiUv@5=ffp`A*TBK7GuaM}2o-jl zz2elTuadWz0UIu2e}b8?=B{|{l$dgKs%t)CN>ZmAdh${(8QYtGtz%Ic;s`I&W&esf zbHc0gt%&@qRazK-qZ0vJ$C~pU5RsDu3;f=VRP&YZav{$vb~{k>vh@`pw&X%R4rfgO8sfV2J62)C;q~l z1vDb<%4k^s`^Tf>lK*$m|Li9mpmQ9@B$A%ZWUHdPISJ4t@qE_{W+=qY4W1E}dVy!s z$r?j!PJAEDd^(X+v=_`Cp@{f6_HK!=Ffg9uz_kugKxWdX@54~sf+emrEcow38~q&k zOGE>al8b~wA{?_3+p?cuTwTO8BEpmPVnGA+&u16N^&)23GcWFm|4Z>Ld-6xr6aUu_ z=CdCEpZ>{igI<==1UolLD6JW+?`(D#THo0dd}DoQ#|!>9jl7xl-Tzt#=$|<9Xu{CT z=P#JWSM1-26I-_D65Nx~Bl??V-!g}~r1vFgzBd^EZ(f`|fAzv%xNXa5xc@mEofhwZ z4v$CR|79=Xpv;$IooSztNOT1>!Hl?w2I=Fs+J)|~4=b1zwC7nJNzoLrlvw3?j#ve+ zL93Fw?4<$1=pX528|~rIwyc){i?Q!B^tSk6D}_~^K8^_K&F0;ZPk22&^Yw~Z9U6>z zCD~G5y3EbE7l~uV2TU_si;t9=HQ|>so|o#uMrY`)K~?v&^6FUm8FHItl3HjdkcUNo z|IgkC8V{dQjj-I5ryzxZ<`pxOazGfy670-1B~JFy(#3M>H`%tVUnKEGfIQwNfkR{> zUXJKjVkz-r;YM70=FbQ`m_q_S-3oh+f72u*ddS~=6Cz>l=ozOJ6akIJl{x7*o|mCr z>nqWPRog>i&+N=t^P0qtd!6R(@oqN9kxK&N>P~jJS{~6gMczUh91$T3-z_o2zdxO3 zzVu5{2p&ICjj1dRrRw?@4)GP`h`22xL?JF;Dv1+|1r}XvgTU(+Rcnqiy%AKMTszj! z*?9=R{AF+W@)J!j_}XQ?a!@bmD78zX>xR$eBZeliVXY|5x0$Bakj$Ld%;{xJgXk?UvJd@(fWYSbphxDdY(@j=VJ23WMfbri{uy|B!LW; z+g#PVWFNg&jy(UOt{beqbyOY8wl^9G?h@RBI|PTI!JPmB0zrZY2*F(z!Civ82Y1(n zySoK<3GT4^t-a4V-`(fjeeZi?eD9xHqk8n}nl*pZy1M2ph|?26ZXVc7FHV7*G+m@g zn&N|$T+r!)fwpSOd7++^hSf^Bl!zyhDblG?a&)*1>Ii+~s>2~omwbT2N_Vz7`-)#{ zWP_!)c2g+fbnZ7&p0HQQs+>%(1QJaYP~L8pB&2#b;Jh=Vp0)8^m3ZIZ$ik&fcPhTw zV}x?cO zQcaK_z;&|Jl&=%ZCPtuZiToKc7;{L2yJk#F>|dT^YUikcd_k{WN*EME;o<3Nj-@zB zBTKAvxGO83)P#m=M8C@HIUdrRp!StQH0y{yHlU5Vy0i4mu{4LTG7%jswRn}OUnu8 zo}zIGC)}mvfyEWU%){aT2QiewT79`%h`5( zPqtgU*Wsw)=e|OU{i^1>%eDjiLrbY=lu@J;aX+A$BBurqZap?%z~}f?gGs9cRSI7j z8M3E>_~uYdDtI9Q76VdL%yFr;&pjC3+^DPGk|`Nsi-Ap#o*Kof@Vb9433QY$X+$?Xrjq)UozQ9U#ia7roy3Fa9pOL2~9Pw z!USjkr@|M`U~jUlxz|5ar(93YTBTW5;>N+g4 zX`HxF6T_+&4bf;!u3`BprIEN^Sw#kT~f^*Y=Lyx~;H(At>HYt#b5RXwAuX zJJgHL`A+jnkVJb9o-$dUZJPH~j4{#FnL#IVAfj*qW?ZT+kC=WI)>EzW^wZ^oo4@ok z5Vl}9U*I_&@bePP38(}6`$$*NG>#6S-2mbulJ-0KLYQDF0;pqLbI3M)(mDg*syu!M z8WLYTopt~i|4Xwb{iOjgNQM5L_y)jxI?aV(9|N~IJ^+V{U{v^Ic*xYH(R8j`}( zY4=tvz|APo{bLIJc6lp4JDVTR1L_r=E8DtBoiTdyJT+?5`0O~IPh_i;C zPGw~Jy*i*9f&XL%o+a*I2>^!bB1pEtvwuv>=Z~AaV<(Z0!T-*L=Vi2$xqtHaxndo3 zn&|^E7lrwQ<~Fyd3Z?rDRP1p*dp!)xv^M8knf%%siAk1(61Zh2W3{Va% zFol>;0H?GsfJg89HQJqlJ&6}<_D?oaFV@~bnD5pQ{c?@}Nay*xXZ`gW#M(dM5hz1i zC%FMMpddpxvi$q>{ta!dO#mHAz!@kRc+La8eF5EA@?ZRgBwysc`~%v=>X&~Y9#olH z$pf)5#9deia(#hkS6h47J816I-=HZB`+wx*ugL#PcK#&@6>MLHP$p35}8wl!qKxIO{JR4vX$nA6|7Br5A{o@{wfF-!Ujt{oC-rBvZ z5Qg!52`SkGiHg8Kp8mTK0=yE#*C1xtyTEI(FW~j&#q+rLKOPl!4#tD=?0z2S1Rg6z zp}U*dJ&^y(+W$?t{(2w8DNy4LWu4n%~kX?!94!`43MQl5eTMzH^5sK*BErDK4P&RZ+n5HNsJDkSp# z8Tg+m^$>;_#6szydotJsiIS6bK7;=v4u1#o$r(uSPmrNR$;d$80c`#TlJCI(cl~vR zP|KJxA5|#btwEvtA0bAESRMTfVg&jQpza@5Q$L#*<}a4wwnBHgkrHAh^HeSl)nBOC z9#Hr}-!lIVDo|?n=mVJJyafgQ2&x4hAh?Ije~9+&&+~s=|EGE0g1G}N{>5eeRXP5{ z6#Yl{wt{Qs2*@Sjd{t0>+XZ-|?EM2?ddRIKzZc`v>Eaz26LOm?5)OF%msb79L;lqT zc*Rqn0pT-n=9M7;{-522eiU%q4|vJuwx!r(onY=Nkm?gG#M)F%P*&bu|4|B7P%UL)3+d^hTJwL4 zBM9n6515$#@S=bC$4OhHJ<-336(A=8WE7)lXj;Z0n#7N{?xm{BD zfpIFKUYONzXyBRxjg+ELZvg}-%j{GP;RPQ4tQxYt?-_Z9jNk9ek^duY=hk0Rq3FPA zvaBEH*aH+yxGFAYs=@&rM&$Dpp!PO1A+#TVaTYTgc5vl*Q1Ua38>$oigRKp)AeqRM7@#po;)rWHdU!{GHg=%n$u&Qi?nz+B{O{0_^Q!K9dym zy;cXI#J8n;6CIR>BkqSchO)oPSM!}`YhIED^6N}|3DV}=JoAFEIp}iI-$(`eKB~um zJ__c&>2`}jQMz{!ieiw_pHjmRwuMjkR4Hm9uN_y;e^X78dwA>^Z+5SNRi~jqKns(( zz&B8#)2inEQ#jquBepe+`YG^XJEE~CVim$9Keo1$Fn)2{c+E9xdtNd&()hjAE8~8c z1W!VKWczSUp=-0s$!7$HtG+bYzkTmBuk=ehj2`IQ=B2GsrcH+DF}!D56L}a;6^#mx zi?mT{Ku;5X*tz(^0BKfXwn~4}uS)$mI6cx(@ql8{*%cQFm#G@@LlKp*hZY8TXk7(= ze0|$LY7LEJ>#n@C^SerXk&0;rrc}2^nY6~2;is$FabJs$9FB{Q5mm!yE=|On5}bhl zxY&()9o!(-iZ)2tmfd~|Ut=In(C^=XfcVzG>M~eaa@tPY`}N^eoMYBhclLZL?s30E z_pbfS=`HKfAfcO44UC zp8c9&vSE4T2rV25X4f2`?p0Y8Tp3c8fF|6E;pXGg{l)Ns6>+AFKXQf;$NdqtAgGA@ zRzX!m)j!emC8NK9iYFlM?pimSLl4T9>hh28ghdP)kf3GMGC3U^pk4#Lnpy+(h!A~94MqQhy)UB^KH9g@Y z>YFL&7X!wkgQo9hAh1D0Dj({Z7*x_|Lbc~7t3KcnJo8!HjU=t9tA6B+!FI)1{a_oxwH#vP z=tHhHi;1aWk_<4D&f$Hu<;nM-cRwdy?8Sy(d*#|$O7fl!KICIJeNu?KHWf!*8prM^ ztpB9jw(2B|BK$W-+|M%zsdqf)T87iUb%{yLqR zCQvb1=-tqzY|QZ|5Km;HBZ`-CXUg{sK%FKmBbzmoQA=hyh^Bx@Rz12LZ*@?{D0?yB z!w_3c6SF4}5kj=4uTaF=gvN1DcZ%I4>7M+UtZ?(#pU)HF-0ZwNi!yNT=fHvrLWy5M zOZFV9qSZTs@qJmx4Hqt$(hm<8?|z?8uAjfqQ!68$xp7rXEZ&0?K>SoVL)@nq8CCpc zv0h$$cxc%+d`IofzF7I#b}??V=}qo?1>_(}aYF0OY-?T<5vydkeGRJg9iC>*FElM8 z_R%e~KZ+2s+zatfX0AVd%Vs9L%YVC{M2>GYgldHI%Lo*XfGT1Vzze8Rk*sPehQEvH zK{d1IRIf?ff7;u;2us?mCi)%7LYBEnqM&HeOmUWB`@S*QNQm`{;z*2?M}F8Zx)~>L zj4^E76O*g!>Y`jmKahzc7;c1-gHwW{oS3Bps z+tCb3QvOwT8pC9-Ez+{$mmh9oJ}qjvJw3fvcU#8DjPGYia_ejv3a%P&Ucj!5a9L3< z+8Sod)7{V0YM0d7XUcEoE{Z0*g}0HTi6Xhv!H$WB+dNBK`SFDj(}8sQl`B%_S=L+?a`qad8ty|!dlT%=I6b@flmjByJem{dC6_lDaV-3#0;D(ilvyrVq~wKl4zY1kOT~=u%^W! zy3qk2+X9G4tqIo`}{f~KCJ;sWWzR^EB&Hu!M=fNKoPEIZ*5huqnMBq zAH%4kB-6I@YOVU4q?UK`En!7(e2QRu#AI{RG14?D3M%n1!mKZYgREOxu%6bU%@$2Y z>}yRO-{WxX0^HBCdL_}PgkF{Q9hbk+ozKl`(RI`go zP6i2v2$yI330J$NsvN|3n8xMwpMq2~Sbt|{}g8OJBJL&b8o;y^kxls2eY@8oyyPR7ns7=hwK7#W#Wdx2x?#soMZpp=P5;Iz zMJ3J2>$3NuKh|$_mVO|(O3!FFwl2!4?e$Mm z^UCspMwh}o{J9x&iF5|!ue_(PNBF)z8TWK$n|O6hIN14fUwev>6nLlJ9j+Q)Mn2^h zaNd{ljLAE7@sZOsUQ;oNkv8|n(4%M7^xBC_b}IzN!)7M_pt-K>-@eIePv?9ZlEf zSs>goHdU4JFgh;Jfye4+K3 z%Ow$0qj;%dqPA%?aImi7H#;B0$zq@()5LJ74p^uuz7o~wnP{wkDhhl3dq-c>Tb#Vw z;_BqgK8TOMm=Oygc&s6$TX{#{LHDmFYRz<;qjpck{LguUKG_fHnc^8z>dfc-f zdMA-8=_J>EtxaDF`)tGU=oqPNMPe409>Sz+97DB8wpW_E--%EU-xbwS^idm@=Q)%A z3`t!{NaoPUi>qgcFLYnddE48LL^<1s>h)$b)tJR8LE7yO5SJ(Y_2~uW{ln)65(@{P1Uv3Jab2F-&KQJ zw48bf6JnUVM|tmXU$}cL%IX`(Dc1n+v)6TNgDFS(i*f3QEu^!euFo(zY|O%Y zvur5w9v0ULjZ@lU?GZbjMTTF-(O`2N%U1U`8bxwtuinlcbV`wrJ^Kak}_H_Tk{ov`?oyevUp zs`PN&D40Y4oPu+Q>tQ;z2Bh~aN+1fJlbJ>iJIG;TwHdCWw5uST-EAt_-&G_sR$miF zl^uB;d*g)zJ#b{ie32$2wSLDco3a9jBfZ|Om*i!8A|s+ZhhY_?1!c`|I4*npTJMZs zgM%^2?BcPp)TVP^Szg$1*Atjip7!7mX+2-zVL1Hi!2eWM{O$D>sy5ZLbe1v9&Um$n zG{rPm_$uNMk&bQ|vs@Pz9=gzGo+t+24ywKB!-eHN#YRjby84$+?G z*p?>R+7^5x*Qbgpj>^*t-@9Bz^Bx!8T;TKT?5f09SCu(q==AzZ#8}eA(*9 z1Jd%j6uf%!%@^2@!wU{^CY!7KvvJZ|A#Z*dGrqY#P`tOyEB{=Ao(v`n!MUPmS$9#kXtn6t5rgKY$kv({f%P>!69Ye>I@^rD5yY% z%3x&6Q+X$yFULi3wCdu8KE;DvUAhmtA~|_(@YG!>2oKN`$iGByDgW$NKDm|3?_@IG zu>aJ3wdhY^B5jRm)KI|oE?ncglv0guv73f2hYh|itsw-@;mbfOlOYW;>LkT^L~oKb zS2};lp4NTfU@5Za+e=?^Je&f)6b-W1(uPZ947hLjr=+L z{jiiG33YZcEr-n@Qo6qEObMx>ok&CcPAdjI6|{oaYLgXUjWyw4W+KAtO-^p#j&{1# zuP`p~vW~!;LVIs3xbYi;?0&AQYfXn)e#uXqTE%sR1iamIyNt{hjIkB$O+2$AZyF}~ zy;MG}O_61loQfsm1{WIdP>`C5$bhbmaxfaWPXvUV@PIN z)?Yzf>c=c0M( zO5bdCTboM>Ndq8|LJf2CFjv0kG;@=4Ai`yy(#^R<(8PL8VO-(uQ-$(*X&SWBdnkV>UY-fWCp zZ>)&Kxo>Q}N_NzuvX;ot#i7p-VETSR9duSCOCU%lBLoo;h0a#Ee;< z8ztkC5mWF?tc~oiw5=|IYdwv^3btdCUSUPp%r}AMvut}FG4ocfua}YoOy=ef6ZBh= zl=8(9{OSa0Y0)d1Wd@k<=j7;^#Bt65R2!L#!8*r`#Gap?7r9haaiegJ{al=-8TgqgaeCe!;z{iwKZd*GwDW>%@&{e_d+N2?DK z%_QTaz13ECG-e0CWz!K+6F=Su&T5-Hl6fVW-C2i{A7u22s-Tf69ng#$o0#q!8k(jo zKo4fZ2%lvYq^(yg%lxo@_(9uXym)z^Ety7bQZv3FTVSRMrR1W^T${EmC(^nYZzpG9 z9&r(DTBWzaVkW+OHf9EhRnOUy)Fo__&8VIXZLmTQUENDTa38xm+o)w zWM(;*{rzwgL(1;^VTQKTp|3G>2uZ*FCyXeBF&nVLC(PmcCs|q5SuWSIa=Lo*PoJjd z=Dg_|2E)B(dO(XyhbP-xU>j!-_H&Ait%JpV`m`)T&WOos`~$V*jqLt@)5g7`5;Ecy zxvmLwK5v?~78(8+EyZE^5CuBwNSr$}+|49Z^n!QMyNmDeVRk-j(`@mbc%-I_InR-v zRz;9w&5ykvzNl0X3WxYs3euI$y^Y$tQ=c1=kcGrGtkkg^g;)(>VUTm46@(YWqafZ! zjq?}ew+z832NV&@eDHg>q*SSarnWRhXB96Zyhy4VBoS?g`81*4Y9c;y^qV;JH4f9B zS>;c}Np3+Z$&_g2jzw2Dw?(E=#$vZM=Q_TwHO*GypWshTanbme>S>M2Gx*>HJ^E%O zJ9`)^G7fqh2(HXqu(-ixyW+bsFWn9pc2UDZBFZ~Nfr8E=h@A>c zEtFWxVy?>^4~XVCdZG(C-IvXkY`C1I9Ln~0xT0xv{1Nd<=d>b1F}sPJq`_rBav`yb z-}hJfLrnM$zP0ariJofj zP&0U&+rKkAvg;l1yTUG5&0^hep77n?&6@g>YT#r%7V~kPBVGcd@qS!H6vk3eH4YW0 zN4|}*Y`ml#uVnBQb!l-K;*E4v)XCH{Z7k&7vZc&m_NWk5G!2@Z=T;B7J8evF+Zb1m z*7M?0%IygM zgX!(n-Pv7%_hH12FHBu@p)2TNI@1TMqnGpbD~E5$ERr1taM`+Hhm)Aa!LBPTf*E&| z3#PfDGB$QGxpTXhRyCJgqBSdk!$;)_iq=3N?G22 zk^A{kj`rS7Oil=0ueJ>a^blKfV7tGTQyGy<kqDt@9DV$fuwS` zfTH5X6C`!Frns}{P3NaU@+FkHk>zBVa3fL0z`|#;y7t@?U8~kL?K$nmwdCspB^J1z zotx>eY6UvS4*DoGop`ut`AhlTlU1Vt{@l_&OfNqQ;noynoY&Hzc5hAP{yAgVOBKVD zmuk~)@$D#nO~B@)8e`?`rj9D|cl6RLOKlIzJ(wCR>iKFk39UwCB4}!;58ak%Kernd z#%JW2>>O*_(gOLkHy`nR6BQ-$Q;oHx3g-v2g9+iZq_FM!yI0^|2)*S&5SoQWO6(nc zaU3hqpyu_2M4le?Cg{Nz|DA-t#-|~lHqyz>39J0!14I>>Vd~O2Oj$ax8D2ks(t_iN zcKz3>V^U;n*sM*LWcx?iof+js`Vd;ZMGd=7^_b}%B11ttX`x3DMEs|(iWNb8?T`dy zVB)Q-7+6Wp*MTqJ2vZmRx3-_)YDiD>2#T3xb@&_guw5a27HYIZHqDNb1d8^j`V{!5 z1u@fhZh#LLpuo1#Tm?jldw|tdmTlnIaoZU%evy>mEh%)#nBOv06Oah6cc-|IckL7(#+Bbs#ca}48ketYWq%oM9YkT^cr0607#@+T-EtL4 z=Suag%5LJL-)?)|Ju%^Ii5*H#@yHW!Kl!-kcwA~@J&(8hzG3z?&&fC+2O4_y#PExs z%QbV{@d(>7tVtTNO}3rQdRp(S7&U(EtCP&hd!W~qso&U0FJ&C`X_jsG*{~oIzYp}~ znHg)bt`kL7Ff=97qr-I3!BasO(US(<8Chg4vyJua_&f1Up%Q>au2N)IPodgRzsN_>4 zEp4Suo00Q3R_5zLWN-03kaCc6R0^$|z7{Bz3yDf=&h)5|KHrbaawlAu{P>)YzxryE zfoGwv7hB_-9#U`Y;j@9>MhnkX=wsaV+K0N(?juL`q4X|I7%PcU_#ZHyJ$$S7kv1i{ zlq*0B)20)Ey~0!AxNv>-1{(Q8hNG(t^D$ke-{{34AJ zy}*IFeKXcW9q6L?%jbAK)`ZNVdL8+s>Rx%qQsh49%R}ais}XMQ=>fU#wy*(%Z2h;0 zd#57Ddk5W<8;fq`>yy8u5M}7zs?{|&qC3g^m>AoLlAchc;p;G~$Yt@Q*c@H0A9wdp zxA}=%z5K!8t6nV>(Js|1{CIwM26ziKx_Y=e+O|GI)=QCzhf?M!r_AbJNs^@tSRGPj zz4$d)4nxF)tMS!MlfUNMXZV3Jh_|b~owuX&`DG_GSsTRroqf#+Su$Rx@~hRi1ObVI zsY-pG0)zV{ypBZNz0ERr&8gD8Cs)c5cK@cvYJBMgjaa};Qb20QNX}lPg&j^C%OoK0 zexRz}Od+J|oeyK2RntZi3UB<20$5fehs*P-cok{0S~xz9k<~Y~88vSj1|%eZT1tX_YDq^IPHH}_WBIodZdMiXyplTtx$gOn~%_J^LaP z6*VhTX9Z9W(NQkPe0I=%%T!`m|c_UnplndI!AgY~>KX+t9gZTHqdG zcZYfqQ2EUIW*Hd~y^y8O*TBN_)8$L#cC^d+6%MBpnyP+TsFB==#n-!*Rb^djNBnHTBxb6D z_|iA5NGp?x1aGr#6oPd52TDH1m=)U!s}J!Fvh$Eg^KLyDA^bSfrU_!HFBut9Ge6zZ z)oKcl3DmJrHfhs;73<$fv!8p}7JE@%XR?deR@}P0o4|8E9;Zt#xtIjyKXbEucS|jw62P8Jan65>RT;dKFh|j-Q_FGckQNiYh(I z)?n*u4NG@cVHeCO95VPK6>cIK7GZ;HEAnXwznV5&=>t;7Fx(ovwIbVi<4jxi!Mlnf zywawpiKh6rW$$b?tH-!pnE!AlE*!Zao{2eY0=k6P+-N=}oBG+WED7h&z;eJ9kID|DjQl z+$`F6R<>hTVuo27S6+uZ?8%^ zw>fNrYczN=KSjn^^6r2K+P(ISjt_!&(4t2?ngN=`)gX8_Okcx%nTXzd?08p6LBAZ| zz&r-YlFNxtonRucL~WI`<4L_k>brTxJh}U(V=Fcz5jgwzN-DkE3Z8clrSf{ZMClH9X6!vTc-1+PoE(a|+3iOG$8q+mA|*g(1M) z^2$&@XnEiIw6zE18rg6Awf|**JY_Iv*Sotcsts3LPrdu>c^Rt6eMV$kg>fP@9+NS# zM!E&93OCmzeFcOy6OS|*V>wn$6ngutuIW6!1f|PoMl`t}tJ+vo`f)3{yr^gpv^~{k z#;Zz!Yx?uhs*i3a@Ex@D)Qt5i1u(lCxJEZxAYLqD@<@DpDvH(_3komG2GLyFnQvU% z_iWT?RPfT_lfMTkiK~1UP1G}&EPTUnh%Lh$p~ba5BDP>&5pDQ;y(Ce+ky)S4dlo3j z86I&IkcL+BNIzg>S`tA`2&2aG8gdx~9+n=j6oeh70GQM#+nHm+!d#&3L_ZQ@^Q{cL z4RTE|-?#!~+BtUx5q7Csj99%L3XswXqB*I-7~6lZv1>sVKOjT<>FY;5H>JTP4;})9we{E1Z{bx!96ypuWjX6HR-rBF&N@}O zRk_o}xi|Go2uBB?xM3+|rJ|Q|(g)GWvXtDL!sG{Qarb{P_Nwl!y`q=R;XCWbz^I5p zs+YF&Wf3Av8A&N#5#H^W&K8;Z5LC;Q-*s1Itd=(699M(Pr^CC8R<~WxSflr4Q&&@X zTdTJVC7F(|Mm0EAVtoAzZ^X;d@sadCnCksmtY=Jfaw0?$BfcseknbS(yJfv=i_Mj; zqbt3>ZJp0s!qu$zk;XI1|bR!nH-(I_lVM}8T*brlK6yb zqTM-|8ntIrHgAm%>$jNZ1227hmz46@N#Yt4xN4J%!}B&K(7tZN76qnD zUXPD;$Ec$h9Lpa)%D8ki3r8?yQ&mx$!W*R-TiqbsRlc3 zgX+SvFKnRXNeWJq5Ppz>uV_B|N1i}f>J@RUx2wX>?EVR4%Av`c`-;>oSHB;wOl|6J z#8ev$S&Nk&$XRU0KfP&l$V)ZdBdgdEYEDZT6OmU+b6liYv>xe8U5RdB%Pfx_%9cHr zs4FD}xX-pqN^?xL0~8kcyc$O(0S%O&i*RB*S0LkDF@Uf67r|quxf{O?%7+{}<6}$Q z|xVb9=OM|wXzBnerJZ{ zfVNp@hlPC}si>X~`_TM?VV~rRDDF*@@v;8ofu!s>m$v{_>KgN-F4{C?GMkXJb6LQ` z!vpVOuc~a{F(-`g1A!Srn_J>6nm098IPV(QOSKXU-cEai%@@`eq#45}ZgdTn)b@=q z&c@tl7x6E7KeZ$_5)=DW*}4idim@mW&2q;kbPT$lS9ytk)PKJ5giyCKj z#cUgD5ZZ=&;~Yfq<)|pGyS{6->+WCaOJaFEIA`!>2fW3F`h4518qSzL`H%~NtexyU zgXZR4n%S=afs9AbE(MX^AHPc!f$$M>)Y}s#m&Mw^&p5BkpqFklvu(0O`=aG^u%ErhQYbygc7kx+dZb9mf@dSTPfM zIWiF=(MJXn4W~33)w#SFzY|O@5>~GHhalbcOyBqZx21}-T!O*`NEYjjS|9t$lqueE zL@M0)Wf!72B~ps-CSkfO!Via|d%YfqF_uo_DV5ZwZY7?ty0#beb~D=ox?(6+QM6ZC z6hEgwpYAHZ18N|fz^%~u2(eog3rjm;PSVP%HdD&xv8@ZOUlEDz$&Cm=wi2tzqzCj+be>htp zO$I+Q>`o*_&W=h2)a(!=WNmlp{&%C>7?8D7rxx7ppvcs_`8%a? z3EaIK-v{s(U)yilY`^pser0+h_}8cME#AMBg2JM=s&|sg1$_*>$ew@8Bu@r_20dGn z?lz=+x>Y%)+e%jW5*2e$39)s?y?pb#K=gOK2a+(KUK33#?k#11JC>2#y)Mi5@+oG^ zU&M*^0ZpKYcl{Wvz$On7m4Uo}!VAbH{W`Dgsl}_iG1Kkcu$D_&^SfiWU zBhwNXAG`8PpCTzuF!$|Q<}c8FKFdjyE9AASE*i)t)#^ZUep?Chv0mMGt}F6nu}UwX zPsuecgH6X^3-pd&|EQ?n89Aq$O3j!LU!HS{Db5d`O^LzzC)K_Ee|6zPvh3zu75b#K`yC>W}<{YgI+7r z9b%Z?kzJ42?a34I1;^N&ig?4Zwj90C>f{_lRWJnucFp>z&PGTGjeTv`&|X~WK1a`wGME!8-sCboS{HXwC# z3m8{CX#rXf>uL{YR^0BNX?6{_F)4Cvuhv_(JJb6xbJr;LuY;d&AG>LmJYVNdz9iBU zU0Lrmxt_H&69ny5kC7X;j}t0~KVJ2fS*T zUQqeXkNt%Kt~eVw`ZWssyms8y1Gpy8%W%F9RFTYm=`az+xdft1fFD=Am&tpwUvH6$ zh$#n&2R2>@rUbUzOOvS{8$8008&-^`3+hnm_l7*#is{kYx{{)4`*QSohqpZ^Nj8he zQh!E12@CFfMgN5`7I}&OH(um?2X6pWwZJVd=No&_oV@Pmmy$-cx4*mD`^9h1Z=gR^ z7C`~48Dwb`zr!d%qC#(RK+q15x0*?o2ATn-9iLd zb?ZIE7}@dnsXMQ*WmY$|Z2aO(%aHNrZ({{SB641+*e3CRM;iR7=_u<4JrUrP^VcjR zCA1>nyXH^G$HQ)qHs>Rb5Q7%3Wyq5ZE--%G3Mptg1i<)EB@d%~}+6KZ|MbL@CmPjN9ln4WG--Gt1?Jbv@p9DzG^)|B=XAPn!;7{cQFY$G7)vwq%X4(R7&Gw&`jdU z#Vx|s=`(av4y!=KV_`P%tE^1?X)pbE*wd+Y#zO}-t@6_GWtzy6ki@ma57*y^Pn7oF zsYa|5I;$ZpiP5cu;?#RW2u|b^pFMDc%Gn|ww)huE+5z*#7qjP}J)s1I6eSLk%&5bXZjp#NXRe)Q&SZ4W?NyBy2=+&^;?j)Pp z!>)Md!zy?mhLhwulkqsp-Etq0Ziogfyu8ex10bUPHFsD-2J?{$<&#VbtGda%93-Z_ z0r0nzepi+$4vU!cDJ4k4CmOY|&Psj0oYZ0>r7GVyhvb>cu__7zh^EwulTp{!_O^Q< zSn(L^!ThUH%Ip4Yk;`wR1HGsG-<#r=@X%!swsX~66>H)}bbUBQgS#m>o)QZqJ2FuE zPGR|}bo4BHtXAhZe%XKc#~p8{-9B6py%EDw7}Nf>JKhE}6%Kuqi5Y^ecG1^pe_3g% zH-TYtVZAhzY{(D$o~M$n3TdwXv9Z+%Q{%o2m0pO)9}b#@>BuEbzo7muG`S4OSvV}l zzgBHGsf^QDw~Z?k%cz6!QoTa4x;MR|5j4%$Ix3aJ$Cfz$)H=~F#3k%-2Erg9;4^=D zv;zuw0+v(RZ=te0((gBcB&+mZsZhGe>Q4M#)ZvZSfn%0|7~_R?C}cg^7zoQjT?UTY zM#Y~JPtqe2p8?V+HvmkaUb+LRV_}wnn}RoPq3QD%A1f2Gz(ODOiY-M@IpLeW`Kn1T zftO(ftBYvmP8n}HO6pwhDJLorrbp_IeJ+t~wTg*LY9ROykK82LY(kySnaY!`t9Br1 z}bMG@tWo@Jae^An0Ex1yTgL&g4?AF%zVSRtai*cZSZwf zkQel8!c4a^;bs^_{WM9f@h#cX#5aR#{+~@8TJ+9gdY9&kK6sx7w1(@ z)x)jcDm*xoB*+fpG!2U*@VypY1p~08m))>;y-s16}I$z^}!@OHis{8^Qf} zjZjm|o79ztDy~jGn)az)(Xg=<)DhUD6J{y&$bxL8ekP+c5ZC+nk7>`37=rIOzmSw&Q0=<#k-3kXin8}u39jioYkW0$-c;0x zLVZS};&*}FS@0Zivq$bOHa3c<^ZQBEe>FWb)A4t&Tmz)BmFexH|5C-rAnU7Lbju2t z7V9hR9u}6VQGr`PF+1zGS zJEm~SM^oBHg)bXNV&#F#!ZGvkxC5!C+N~%PSn@WRJK0&)aGnyX64rx4Yv92A70+E2 zUNz%sSD>HovFe@-C?Vs0%D1$TqZ%QVpFoXkh=_n#&pbbjq(7Yc(@XCKuh|7epP2DW z+y(k$ps#%0>B*pB9jK{;7y*|;X1N1!} z#4m4O5x<{UX!!FR-wF|~{p@dQ@ApdY@&)?G5`TcH-X%v}d@#~keRV6Fw7x7!Y+$0F z;ybR@U=*(YDFx)V8D2@IT$V0oNPHH+(Rw~uH5G?M`Jj< zTVQ96=BGA<6fGIYj`d#Gu@*>8^XhUY^M}iOTDr)_IU*>F3%(Tf7hSR{w0{E_XrtUQm#8&Uipc--!p1btgSK!Sss7R_E=$l#o#Ulk7>D3{ zoRCIi&xkFXq_9F!z*<`KSyk3Z$ehCI37C@>ffzKjT;c#w-$LuSNrN0I5sKq`$DNzX zfid~{5HW)y^8I$B))D2L>!R%7D)@>BGhs^za>AfY?v2{bm)p zZ4L+JJlH%zH^fNj%BHISU_VpSx~HCzigL!}bv9Bz!IXeD0pO zHeuUxowemb8cQz#Tt<6UgZpSAKe+*cv0j8`E|7m)6rZ21_QDh-(-)xw*T0S#T5PDF zjZk@phzc+K0)alOv^G5!t^$I~N59gb@k>qnq}L)ae$JAec_)4 zF9a7($n_kZ{I%)Gpe(Eko}WlabEcNRTeuLhoX}xuy^eyy{?k7DV?o*6&Q899xIFBN zQ9ts9+a+geW9h)qJDy45qcE7)9dGH2sd_$`*$CW!77Z{)C0*RNw>J+VNPND4u%gNr z1Sn8G5I&gxKLFD}EWa>K<)@B^1Ds&^tTpN3C?Ize93}=xM3uBS@9`XBMgs?hVmu+& zm?AmuzfT7^4sn7hD-w|qQjI&564!SUNl{Y_u{L-&R7aG}#}$oY2PV}Ck+il&g4bkc zf`dYQgC@?eBu3Z#3fOAxiWj^C@tXZ&mx7hRzryIcXRq5E$od<)Ubnk;cQq3c+esKG zF-AT`PT2B8HWDZC2LVQd7&})C3ejGOF#tGm!l@G=hv!cR@R!gzL7S|3w6Ys|Xv6NU z+r3qWKc6bNMA|IM*ZAMwpmkQPo3^mUT4=TYNv3G*T=QXj9G*J2KmOzCWPoEfB_UWe*OkBQ)izp&FLYU#s`E1K=Y)7D zO{9w^e&|o86O^XZ%)=G>!90+a!`yLnCBQgexVAXG!cNcyw3N6kEr# zD59clRqN>T9<`IZ2)FyFeM*30e~dgQ!EN+d1uwuA_}4hZl%B+7fGbO*BT*DVSwwM? zfWK!ITUgdT18 z(ROzYwS~E8`gVi89Xft6jq&+7#&k>quiZzzE*rYoK#bfs7~umtl|>tkaNYqvUIJV> z;Z-?c*#(gkd6mG`R_1Q)KHA(_Uqk#2=yBDnq5hjK6~2RFpFkJVub~XI2Lz%SGDIh$ z3W7RtoKWhILKG2?>VQ3E2XUEOj3YuB>t42hSU-oXC`yagw2m%wTu@UOHMkOIR98#F zl{h1`a`}Er6Ewi=c!uNna@7FWj@`94X2+NY7aiA*M#4qM4q72V44t3=O$M&sjyv$W zuLnPD^|pqd^ZNA&Mb@%D`lF<_? z)|Vt@jGG0(NC25%FKbW3|ul|TLr!h z>r2)$!}S!>ZwN$bZtFp6;87+EVR`F|*^m4(RLNWHO=&_V^4e6}`eNX6>-O{mkDbES zmnf@d^YFwB*COUI8=cb7+ym?0@w^yk*olNd2ud^%bWKj))hO6Kl0O@e$qc2v_ ziCxe0IY*&DzC&@%effdmhJqIeOc|&8-ZDiUoVXp< z4$YQ-62AO!#3n)-7*z54+|HDuLtdq>b#w_{H1o2E0N?+)$ zqNmd@3D>tqS0!BRo>#-?D15mtjuPx~^-u*|ohJxmBwVm=t1s~dxL6&q1u22+QyYYP z7QS^RQNU=V>jdpZYz9U|PMl<125|~36a@^zZjE9b;m~7$;%gk=!5D-tTgV!1^pWD> zG~l{D`NwBPAmDUQdY{)pNmKN(L4 zwof|F4YizLV2P1guBzP`O??k{#D`A0c{)p?|5bZY#3iZRDH(j8ADv~=mh@6&rsthZerc2re+f#6HiIH*?AQaPp~2&j470V5?q=E zP2D`6tD;Syd*VDwbD;(j=v`@Nq$FoGa6JWIKokx`KiN~^&R5i7eXI^ zb5U^B!NmJHEAT&6pe+Z?xK!eK=7p!c3eWVzv;)yW1Vsl6n5@X~!iIXY+ zB&IDNXRLzxKEI0qUs@Hkav5E_mx#pZQj1S^xe9Q>r`ewZT-5))3a&H%_bglrXKvv5 zlS28+UiT+|A@it*Z_78x=TTP+pZvTB{048qFpp2tR8@o$y3E2=n8!MNEeb9zKvJ>2 z2XvyXD{=pwl&c;YjX1@Q_e<<2_(SO8#ss4XSJj0@gDd$tjV4tp5_%$C)!`cYp%eJO zo5_9|Txq3z`O8nJowA*b)r_uPD~d^ifqQHtGYX6AYdSb7y|X+ZFg!&LqSFEL{8-6B z3(HNy^&~wGJF>^*Cfs)tr;4&Zpu-^#60wFn5-ul*v8HuaW z+yrb^{%bb?;bMX{&I=66O3FWpPT=smS=|S80P+DGUi&fOmyByC_PHP}awx^XP)lX> zQ~Bud8axBu#-zYPpjvIVmdnI_6mlUC)XWc-+rHztJD zcRUZPi;_0?HrnGi7A2BIf}+S_=!tGCuV*-G&WjUe8`F*gF6Qgak*3#=4hl%@v zl+oB{z=D=GE6@aL59o4=O%oK z%1DMNrVEFw4q*>yUlj!>oJqMo!$RRw;XBLdL|Oo;(^Sa7_!&e~1U`Z5;&4gd0Y9vf zY&^*QwoFkJ`x7U=s}$Zp5iUIgbuD`XMzTlyyC=w*CSx2XzFVqKc%sQw-1ySYuoPN)cTsdhdgUPBY3#lUsc2xT0u zhPv>Dz;#k*Stt6@CHJXPBQ>*=x)%l4jFz|=xP*S9N`@o*ks272+{QGH%ParJAH}}c zxq3}4(NU9?T!gE_VmJ?1R&Ha2t84e{-qy^b90Obx7Q=bC>MVwzNWU_0om5x}=i#a| z6Mmv3N5WOB;ZEUFMXOtXqS>E-i=W;fL;trae(MJqF4LD;xNyS#xJ>IAeLOz=zaI}S z-X0zu9PhqAI5S>=8y)BwxKie+guzJ7pTMb1SR~%~9!3;9u{#DSg~o${Lz?H&&J-M8 zXh-w=n;<__v?dR08C++3lC8cp!UeJlc2<+2QiDoXIx~++Je`BnDTPaA6bW@=`+)c) zD{d!4UJGEMv~ANUp9B|)N4}cQ2XyF3a7a9ILZ1TG))>EkGxlBXuBlBV9g}I`rDC0? z-f6XP4M~s=^8p=^k?%SI;;-2X;owN5jUeJ_j?JHghSkF5`IN1weORxT=mG7ziGS@R z+!^zl_#Q_Q`!3cSzbIiZ2i;6ebuCZPMSwKLj$sqoC zI1i^^3s*+d{ea~9t#WG6T1S_RvtQtTym1nD{2nJ`;$<`n{1uzSDR(FKaDv?=y;=(Y zSZbqR{A7%N**)KT$Hkt4n%ww7fINa6@lA^R*SPHE?C9X&1o;Us2iC|=5NKCdc74Gi zq5;Mckjw2ZyZDgMqi}11Rxh&yqgQ@(9?+k0><^80K5$@wtTmo;!-gwALP-EV$No^> z`dD{YQ?`jCAGzR9KdTa;4B^1k8W=cWjMXK5=#S8Z-QNLMMfxJWopm}Rw*G}GU2o`A z0%L~OXfVE2#745Z0I!6J1bnZ%BnTjsybk!vGz>7M#{Pt%3vZrt;XXwaC#yD(Tmm+y zuuxJ19#{mOWD<395}UG78G2E-Z1GK!Q+5oksl_Q`C4yR66qtsvXW6a00)OmOkEV%^ z29CQJLnH(2cum(;*`PQ(p>lYt^Wks^Nu>OtkG-|xzBF+mV%=G=?);FI=Mn^^!?4i@ z2KD^7*G#aj2=Lya-J>J;$wBl( zpbM(2#yc^GF(qF$vusT{wp2{poMo%z*iyKRl%b9=r0_JN46=A5O^yjj<9eHIxuqcg zOD(tz%9Px^Oaj?#y`|#!g{-y|D)T8+v)WQ<8mu*0cuusL>Do*yEd^I!XDdW%7wPK1 z%A%VjUq+&Q_Wt~Yoqiq;6WM|CB^%~&$zL_FaWS_cb7qN#XSR8ShUc0{4Xy?gS&lCbL6eDu zl2+YxYH&4}$ob&PP9%q`ZaQ^uM;U6nTzm15i?dxPiGQfTef494i?g-nThbn9d8MbL zh!*WOEU zGq_}fyex(H98<$qt%MiOs07pG832cqggY}09JgZT4QqgL9t_;mA9wdE#J^8m0|1vg z{wh67Ib0M{bO3axX;K??tr9{zd9M>eAygIqTT5udJ$!VrBgBRaU92m7Q&7`PD)pci z^XQ=!V5l;)vBr816BHA$$-?o(rxYlGL(vTmxDkjM#~HXjQoI{QyYZyVEsXJ=JV2T( zhxqXTOU(eRio;eI`)c*Xr*MHa5ySWCG$_+EWDJRvOS9%h1k;?QbAe*QsRtk zbNTC@-LvP2j8cXxvp%vEFij&4^i&}_c&fcBoi+Rrja#x&> z+&xni7xBBf+A)gBgdLnyib#ZGhkbamyK2V!Y=72kfd>VDI04{M&c)=qmAVcSzAks&I0?z9v# ztl~#KL5%Pf0XXy&G}lht3H*V|m7tdsF2;SKHK4{q8UOpf`e})>7Pzx>gsVI@JWafc zl+V2dQXULq$Hy3gP&sIbZ_os6Fm&GnFXM0^Y_M(gBM!(7RHS6WE`wsZvc-dMqreG; zOwoaQV{#*9P}p(DqMv?>)wCOiOd&(zv&lM08Aq9I=T}O?K6+5;d+xPFK5b2?+3vTr zFSc!KzaXzi1VQy#b6~=b7)Ek6^rjO(2_Rhb85_6aRN8BLp{eh&BF@0|dg~jT*^eUB ze7lc&>NU@R#ltHV$10Mk_!RL-PCV8=K&#YA8Iv&2k0D0So&KE%npdAh+pLwoR=1M} z!e*#}!YQv{(Zor!EiIu_JnSRsU@57EOD39~L}y);9)&nL^!iBtjxo21O5ddV+P-#P zz(tBnb|p- zBaSQW?}Q5@36b(f1GT0{FhwCrP>hpl9HReo_TgAeL~sXNMw21>vKX7~A03LfHxUk9 z!cHle@)c~4Z1fZ3Gn3WUF63N@RBnK=!_EvGW_>&vx6C%5>o7i$Ab*JN$W&b9 z6qw?kq6q&sbpopFmiYN8o5Eod-=*Mr*d<&PO5p(Wu_wD-GnKaeWqu`}Q)b7<$D|7H2!V+Y#r+_V3| zZ-Mr|iEzHzCLk4(DvPF+yZtigmOINrx8%ptyBGXu0Vj9p{l~L&wkTs91&+(*blxBJJcB2pnJ5(BrXN{a&e4$}&e^gpjbA`ql+rIfYP3#-L&!z6xGVx19 zJYhAK76&Uwk%`5XuC@hRfQ%`IKyrd(iWtudCcq))+*-@w&ME9b^*|mAV3gbY*`R^s zt68g%j0xKUDMaiHzlr@M!C{M^E8N};*gyzOlacY-XpdWA%FUN+N-$p7g9cauX?xq! zjn89YpNldfX_|r;=+Rq1F>*`%j?~h{ZnY$jm^We5SQ%sfQ)Ogo$b!mQ zeZ|P78!4Oh6cwy3`>5S(w`5~iuW9bo?xP2*h0rJ4wm+-0EW1;bWmNT6F_^Ncy%Ivq zjrn&<;M0Wf6Bz}bZIX0Zt+6#b9gW2aDi#R#O#7!*x_?mZ{>hugnKw|2Zy*V(wtG;6 z-NSeTp%<~yM=JFG6eG5BcwcKR&deo_3~!;^T8W5qa!+v>B``8Pgw(5vBk@wtV|9rr zY00EY8PmAjEV7=!MmkA*uv@J&ZQ$BuAd^n)iDpc^7grJENf@qaZVP-Q+kfa5g z<2exF6=Y?k{bKOA+I#NEhBd$ELd7@P0aI?BNZ+JtP# z=F90u%2zYOk=#YNk3K}sZ&Tc&3A-%siyqRVkiT0H62|Ha1Odw>b9}KcI_$lcC@dp> z>vOK#Zx7TdkUBxIvIEAU!Bqgh?x)oy$1ktjKgCWo_LKC+M&c2>T~vbr$>DIEh_i>0 zD-1e$QR!yvu%)GP83{s@1+56w68bAgU!DZ+E})7GnpH~-xzb5zxee#9k?5g2~GO1yRW;&T5|Rj6EMQpAoCVmNFy!#^~}j)!BJXqkQ9OTQLcn>g{6IHO^m=v(H! zbIX&y!m=F64AgXA7Z*WW6DRr`8u~c!1i_I;PA&8ms4DTNW>o)79I6xG zI4K!sj$>^JjuoXGE7)m&du(e#GE3Z4&k#5Xrb#s$3s|vqtXL8&IuEO6OIM1RYFV6! zaBhVLj(Y{ar@q%$Q{EH&81|Dfj$48&xbT(ucnOj8HSg;^yGVS9N1V?1^g=2VQgTai z=v~Bk;)foNwf92d>)!QIkDs^VP`hBW5nSVo38vH;DVoJWIKvoSp2?Io=u8A~2OA;f z+o{63%iK1cz+uN7i)IYV4){9(GOylOQghdj7?JLy<$wI+AIp;WBUd==qvcz2JCN_D zQ9v9IEGaTaPDW@B696i^7U#JyW}QgG#OZTY*@B!*dFacEJ#{F&;|e4Y8>%2`f^>pP zgy)DyD{@V*wpxcFBC*G90?idg9c+*6H7=#>cRHHBP&W3r)irK>nUM&~T)$lS3Ku0i z-3S$`N=-e*=QA6VQf8%a1BbEH?QrCWxBLu-KW$kcTP?k&{&EC%TOaA}ocK zct3G`nQ(_y!V||2`e@`kVLTo9$9lC0}B4SX^UeFtB+gt1r2!fz?T zur=GjQVEOQ>)Gqy6vs2i{?|C;W_{G`|LyI~Zhrr7t#@Dc|8qP`NXS{GwMlkYXQ$q7 z=|*e++~0?0)@bQn!2y#=PYWLGn8p`6~!(yy$v*o3^Ix?iSJ2}ijPa?9*Ky5}~A${4?o z+U=IG7gcu3ee~%wmpAA()qT|KQn-fml%f9bTzJK`XJ<+5BzSa({2`(^X=UH0SUe>G z>w||!V-u+V<^!2Hw;w~Lb^|S*axfAjSVF&GC52PsJrKWYp)RB#A$9u`j*9{225CO! zRwFJ6sp}IkFI+9Sa2RH6pPEoyVi$*6tzw$33gxqM_N9*10MEssW z*2zBF>c02I2zj`*Ood*t!Un6*%KDV^I%h%!NYRaO=XCP%3R2CnkRZ2^_)Bf9rA({} zh$@n8gWDAGF&++;?IA;v@E!FjG`5BuBrW7rY7jJ)!f`a51_Af47X(Uj4=se0r=t-0 z!q#f#+y+^}A^j;P)2NTOx3)Gm*~XZLyT*6NBs?V~m1M$8(CCSBt$5(LfJzt`AA%Ut zpA9bcP||?Fvf3xaL()?l30LiGW2hp{ic3NlM~q*9nw^Mvmc)l8-Z2WRe7yxgttD=( zq`awwP7h)MWf-VbwL_)}7(R;yAf)PRK$2Ta!fG;2@(o7cIP@aohjOZx5Kq&sJG4ln zH;_?jQAiW?ZDG#oLm3ThDPkq3I`tEHrTvcl;K#rTPaU>BDz@ zB&|=tmn&BUDbKM!It&XRIkS*&m$%K&Pm3WCEKwQKwJ_rOY&C2cilv9lF|BOeaX=nS zToOAnn0KQ_v{N!Oc{=9U0mp|5aa_>&ZV>yPTw81s_0no02RLzZVpk`2$9?2XyzR}h zB82uCQJ>znT3U8nE{n&*A$F5KIwoQX@H0Q1hJ#z5CNx);uZxnRnL(e#J^>;nb1JGt zJ>rO8LQ%d5XD3)IY2X4a96&sp!p>z_0{@xaSa2Iluo5gwgm5Gf-}MR{3~ZmTYss9ZQWBva5A-ta8|M?&9KSTCUw95W{WoVpHUEn{J zBgC6dJPoxwOnww1fs9gNgE(3a;~k+%#uZ2mN!}U%3bh%6{>FCztIeH#{wn_zUo>x_ zzpB5*x74MF(U%9#=V`TK5;-G>-@K1Wh#weS=R~EfefZJmNX2V_^g3J#8zO6~BV;y^MsPBT#63o4_T5H7RuH0(xy~!fW62 z9m{h^lk}Fb1W4tV3{j$!DberQoAw5P#OIxB_X=r!wXU`GQ2;_MOiM0IDqzr^$8rmF zDYqv0a`@8szd7#z>fY8l{&0S9W+%5vbsshN|Mi{Tc3%F!v%UFp|9_6BXQQ7mzk7iH zjnGklJUv2#I~9zz*Ve%llS7ru}h~xd}1ndSJSIr_qh;92mI!w^SxkHI_rTk3&aGK!M zUtDhz9iB*@O-cfEr%m)vG6nT8c-U~}9LC95Y;7Sf>zJn3F0yXsn#JXG7S9e&e?HtZ zOe8e_xV5asitlKy4?A-ZZbxE3U;n_6DnH>v1^H<72yYYh^A(+GcA1>{CCHsO_!q z*j>A8_xeBdx?SnZ1R_n1gtlAU0)!!ifoBa#eB;F40+8}b;+%^oVGcu&&C2Zb|4HZn z#TW+>j;WnQRjPoR^MA9uxsf~nx3;%lJRk>g?$%KIB zLTv`TfVMfV({n|l$R8^7%8v@gvP@}#MsvW96Zzv6jHmym`7~>AD%9{+vUFU*nt3}j zfKlv+$q==_qty%RGoQt&*@E@iv`;83I?b&*C6|LwC?WrJCPDEl(lD6)Gqbt1 zUBifzP_?riGZmKbX_O2=-z>XFjG}C<=u4UZEo>*ygLKeG?f-0}_CXjK`p5EqT zDRcf~hEZj8G{K^9Ijzjc?Pn7JkSs#shHI;H=Y(XQ5#2_;7QRP!OGI zSg~qL8Bm!cn0z=|X#>u_HjDD|e8J~Q{J-bm2?;5#u?DQ>|9k7*jlBKuc5nN|{`XlP z-Tu#sBHB?d>iaOcXGSmed4w9RjG4+@H{a9;p4G2Eb|k-KqX*=N9u5<<$(muTIQM4| zT0LQ>lDOj|-Nzoy$7M-p9>q%)L=Z(npR>AwB6*veBDa*Ql^tbD(cy<2gBYwfr|fwu zUSFpJFYB);P~)xgT9xJHkU{le$O8KK9=cD~OO&3Ld-C$%h0M+eI| z9%*AkLNy&Z$=Hr!;tuJX&X~}I9jC6j7`>ya6uqN`LN|C|bE88`$ob#AT6y>3?EK>J zWVKGXt4+1i&?Do`EB&n2PH){fsR*nSuk}&OITc`K#$diz*hV|bY$w9bq>s8FaP1bF z1?JXNrCwz zZhmT*+|)WXKX9-$g*dEzbiSuU7x>pW#FU=IWPr2ajFTk#2`71+2yems+cKb{0yr^t zJpbAIR2KTh<((Rh=9YI_|2h}_wK!>oO`O%}n>V_y)X#;F(3;Ngd_&8{B3TQPs$FNa zYss%5Rws7C5k{{r*3c^e0QKL@)`*qzh`a@LDK8yfT2#F(mQTl;7MJf#1M!F1MpJ8= zGFWtbn>P-?Uj_&D!!%EK$-%6? znwaf57Ed{u`y(u>W}Cl&ezPc2qK09Cu8fh{S&c*wgfzmC@9_ zP?O<8g2a74&JK5iJ%2jsqxay?^0X0}51jdqsU{;*HoMs&)m5w>+t-#%zC6$IJW2kq zd~2Q}|L=Bpx_SGb?e*S^{qM6p`2$JLD7P^%M6NxMCB8+qKErG$P$jSF$r+lIZ(_0_44)~I~!Xs{NJ-Yx!s_v$cuX=m$jXh5Q$LI#O0Yo z0xCGuWK3fJcaYiGSFgbn`nqR06WDs1^ITX6Q3t>Me#=5mI*AJ}plv&eR0MJI-a!HQGgzk45hPfe+~x3S^*; z|Ld)9W%vI^cc=Se|Me`-Q}=)4-CiIRKl0V`Es_BeU|}a92|>l+S_l=1;FE_!r;sZ^ zkckXWEli4984=$+$XWn5{S9$4FV8&BQ|W(`Fn~4me|;-Y|JS#=y`2~O{~XUE>A#5f zbln@^#E}&Lgm^Nz6gQvy*OLXvD|i(9*M5LUdeFWll~W1A?Zo(+Rm0SEf;1dbOi025 zmqE&v6WPvzDyBCmBQVC;n}(hfCOj%#NJ5J>g#Hr}9Ds|9f=KIe4H_9t6Lf>YWG_W{R^cyxr!rg%P5iKrq?~r*-14|>TbWR9@O25D;CSLERHoPzag^8! zn1*~7VFuN-K|I4790WX80=u8|(ef#4b~n6R2C>hPA5L#&2e3ByK12?s4BsN%iif$U zDvxuw8%4YEgv2OzAR=Efc0zP^0Bx1`HI1SGPjHwxfw&&&Mc+?p0>wG5K~}D%f_h8h zf$u>KOA%-f#tMPdY$XoQATx(f8B_0BJC#-@r-USL{Qz@AWVt;4@chq{&VOl&`9%5u z=0-mL&(`+V*30?-98XDhN|@Jg=%g;OA$RhQB^jp@lb1xXUiK`{DC-u=9q@R$CW~$r zQYF%ll}3io0xv7z=dyx-GdHe)4Ajy8?soqEx3j+S6945{o{Ib5 zOfdnf&JiiFn)f&B!T_$lTE}GBl`d3ReY1(#Rc|GHL9NcB>GZy8Hl?6PT}( z9$gvFua}4b|FX{<^xq!iV8Vm=HtVBK|GU-8)Blal?e!P>{~XT}I&qQ&$6OG_XBFLy zafk*}KY&|vmd(piw{u+~7oHA`hIzzs5X+WBT3+z19*6vGAf= z;dvUd$K@V>TD1{4b3iI+4icn<4*U??Eqnj$;w&LCZY`lbGMSJN{k(UEJU^x_d*mk_ z_?Ls_#aANiABhn=*cf#Y6HBc2Sm^}S7RqSp8Jz`1IDZzq$e_5J_0 zme9{m?2{=)hx-S#Wec%X%l19&ba=U#{Q5_vi*xM%(}Ugp_XqaGYuZQ6{@>VI&)@&o zw_ogkpXXUZCus^Vc1?u*2;h<5(!QwB`a=?g6U`kspWa?G-funTP>ci8-jKX2uWxXI}wH4t_q@08DM!j zX8 zo#Kd~!*EC-ovcejN&?)16xlKRG zlZb>60nMS5xISdjfxVqZ+=Ed;UsMmbW@nJ5z^+A`D%k{a(I|F+eCB5}Hkrcj}< zd_L{H+dV!1j4Vs^UD!&ZVmk!=!j@hQx>hZDySu!lr0 zE4qfRA!M>z3d)LUa|E&f~#w6D&lFVBF$@CO^Z8km_(flMA%h%?B=5>YA?|- zxtXu2KUPlba{e#A!s~m4t(+jh@q2!n&7S9m+K@>Kt{If zBuRjQtAHq&Dr2FkNSf6s{VLWK`|RR4fq2U*rK7o=Cd=b(Ym3t2YFBebajIBcrYKi* zG^OJxCbxOfq@LooYxnGKHpT53PAHm25H&jG5_#y=lPMja#N?Jygxfnf-JCQ|9xEkF zFhB#JQ9@f}l{Ids7FP;sVdK+eu?)72ubBVEy#HrT{@Nh_Yj-Q3|9O39<3<1bERQk& zYkmuTvAnOXw0jUmOQRUZ<_w5nCDYBulm4-7g>D=@d810>rSfN@ed1o>nCFYCPS>m* z>T>&b1_2p#Fm}8F-r3&X-t2Av@I!C?hac9vt~+!b$JyvQueaRoogseR>!f`x9rdiE z-WD6!F-dSA`9pDKbdVd6sVA+i;x)}K2NEBOp=SuqogD6DN)v7zrmC&b)l9_|y2^7P zYPC7)5`x4KA-+9uY(89lL6xj2WhKoLDrS|!6p}JR@=_;2F{bRs4rj(%YYDCRHnt_8 zB9kWxgFBT#HE}&@fE0>PE+;G~3Y-wjO3;hrx|mX#k`VwQ94V@uRn{v53xec_nA_G- zk;X`j0d>UaGvQ8n!q_x`h~v5>X19dkuGLBzZ%&F^qks&YK%9$7Ai|jOqe!gVQxS+z zq%?yhrRr+L=f|gLfY}ldG!}aTWo$sJEmgXY+UjrB79c>|6>NkmDH>~C4u61!qny_g z-?_?teP`nbeP}r6n#o%cpi$Mf+EH3#%Fj=xc@<6cQA|zY~SCrb3CGsiyt3tdhOamBxDD~b%Or=Juk-qd*nHd&s0|13GsNw z34fSbql{kiwNc1%swFr@H)9-Q4RE#t=^7%*7{@n0#UcT+pP+CW1Yn6uZtp(E0jx@a zAJ`v6B!JZ^DZ`aaPUD1kD&uc9Iig75q>|t8W<_<_)6(RWLnzu1$k$g975RX@7D7{s zj~Mz`54w$OW+TG1ak>E8NUVcv-$C!r&ri-;NDz2o1i4RyxIv6i7_ag|3?fNmQo#NT zsv1&m*Q3srSw`}K`GL+l9{z30zDGEo_>^+06M8&1F&YNWNc1D9oM>8P6*33>zrxCD z7o@E|l%GH*F5#hqJwXoCR6sXVinB`jy%>8qOnfJxvklwxDaZlN&W==kTI7zgd&PV5 zLwJkNu6)LC`@_3z`};CeCmT2{FuuxFc58HgbcWd70nl}Y;T#iVl1spKLbknzWEy&0 z4Ugkw-;c$*2MbU?z;LsGRRl^N#CB`~JW1i?gEeA{X5{%A$CyTh=Ne~FQJf@xIHIac z)`{x#3g2aZyUw0oL>Ps0S3r@CJL^_>|Hun{h?Ipmw{`+4j9?jvAQ^H}v};J%D#Q$f z0ML4@u1}v^GM%-sr!{2WR|Jvw(caIX>|^z4nE-GdDJw|6ziOfrtibTk&nP41`>r-jZTN#=t! ztt!^9Hx)YfF_vru&nX3Jh zpU5m&tewgv_?i4fm4(c+&hx1S*&!*R)gK|ZQUP+1+t@BxCLIZn0xO%Dt!SpMi4b&Q zjuJyWpx>s1dvr?iQy-=HZUT|KQKqZo)dlLmK`YWT5Vg;x;j4H>5%$5kOpB2tF35<) zInG;kAVoRqHz-Rv3yD?=^=MkL*v;yyiCz^d@-iB2YL}NS<`7i%gcRXBYJW#)Y>C_gN6c`8a z&+X!hTW!41IZ%PN(yv}q%V39X9)~SsYMBaVHj$k-Xl{Y&)9qd5if@1}&Ky4rwm3+$ zAS<)fo`kDhKsPhV&B_H!kY`M_`5frWL8s%};HB9>wd77 zWqpw_PtP~yi;L&f80II$Z>q#Z-sR1}JQ}8KUXQ#=P?`Yp)Tzr4WI-WjzWTVTLdMJ% zl|;VC#Tuy#6|&&)9t#fK@syAp#XvJMX3P_5tW*k#eQ1hTqX(A(btv}UiC2OWL{2h3 z0naB`Iw{wCjIpN~KZxKtN;f9$s?W}jxKfO3vV@xO-TC>6;nr$|JC~oy7{Mg{>ud8T zFVB~K7RUdW+XPng|GkZkLj2$DovoMnpU?6vi2t8Ab@U7evEgdoFrHiEQyF59$Ym`J z`unIYBw`I>tYT6{h+xipG-#q!Yyj3sN5pDjGYm;|DbX{-#| zY{IchU`?zWd%Td!#KPiM@z#%N@{6C0-|;Fk)_rZh>6+VtDdnuFO=Z$wSv=v2CUnyq%4N99=4-Z)WetDxLM@|m<{QXQ zt!W&*d9`wK`r#k{y!d!}lr17I(U6;&mHKdco-F{3pv8#En0W+1hslkdwWdg0b8D2K|m}N~VwgUNamLzCK*1Bo=qWM@JE?L5&SAJw& zLz%h3tCOGT9fSQHr0#yxxpv}CJPkYiib1np!@2lK9jG`rh&eW1K@;##Z#p<}JCp{% znDK?H{b@(wJ(I0>h3{&rSy$Lx)pZ(lYPpWZie*}8mQgR0U%WiKcuBJFvQ*!-nY?9B zY!~Od=_3-BciedR;A#)M9VJ zs5GH$F0~ab1k3aFO#ga@>*7M1W2dA^zO}NtX+(WNhxkwj{NI&$Gz0$+m zCD9$=kS_Gx&KhLaAKaZ6#oLOi=p_J26Cy(QI|$FpTa%0wTO) zgzY6?cI`qd3`u?T>VEIw^!(!O;Su-bek>LQE1OsMe?RzVWid|bpvB!nn_B7g+u4VY zr+Wu)UabuM&=X4=VD2Cb0x}K}4^_}ELWv#_WhBL7obebt9eH!I7Rf-cb~AkU#409WtKaObP!%hy3&Uc4ONP66YE+2uIjt;Kq=qM=qT=9|XP;JEOsEF`Fm#_U^gOzY6B7b*hl8h3vO!{8bp#GaFbf9I0R2 zys7~4BCM+nX!uITRR%~FVS`jNZC~27>g$ZEv0;_|avsYnWxpg?&1bT(>X}WV=C!aY zY3?Z&R(Zbc4-^dV-|$%!|5p_etd9TN+{xSjZ*O&9;{QF%vl#v_Xa1w&7%cF#Sb+IY z&C!8Mtbe$J*Zjsm<=rm8_NTn@tmzMG)d)^1bctq7;)1P+!~&?bAu^PEZVMs~LaqH! zu~-eDGZ_CEnb$_PKUvl4Q&fl?h8n{kz0mAtKauj4dz#UYaRO&o7U%0VP!-o*7XDBa zQZd&uPn}v_k+!Z>;<^%3C(R8!W9cLuX${3BaW15VE6FXFuS$(%s`0S@m8Fdf;Qs{K z5-k`Q!5aQ=ePc6c|Gm-O*nYA9ewN3a;7!JcESQ1koS{qD!E`a%E{QP#yUk9X^hsCp z1{AQy5&H{J!V+KcQj#dOd(zDZNK%@uQV+@4-27`6i^|A}orya3FQ_}l43(dt@aI^_ z&+|Ml`TuJ3|Jfw=N3)~>w6Cd4Jly#sNWN#?|1&55tM~t0-`L6ff39~oUgEzz%d=F> zm(tpmF(G9NK8Iotkx;$A@N1Ty)ItVuLUfA-$AyKNjx{C>|*ftJ%_J7ZCsmqb&kXWWjfDyOsfiCx{($DOfY60+E)NESia zQOB9j{y(@_2yP-RD~gi17H|SurrZJoV3u~#j^67Y+Kgv z9DEUQEMDLc{*ppF!c^iV;6_|}=1&10)FA;Aw_vaFk8qZP9x-;|RofP^uk6%Vb3_uy9jUZE*3Bk3c1b{7*~zvS=Lfnbqz$>jF@dZ| zIHr~fkggB51>2rGUaBWJ9PXJ zPa~h8JL?X$SwQ0d?d)F=VWhg!2;DV=rre2f0Yj9fp)DU}DmpJF9fRUnBtv}Q02$1e zxuSRRJ~|SPJbx0`4Syc~qC{8 zxC8In7;+L4_M7$PNf$x~Svbjj{fRJlRZ!)#az#{kIu^yJ?d0{?XmJj;s*^7kOnt))d|-XJ%ja0Vr+fhWu~H7`|*N_2R&Ka8n>0~9okHZ`7^Ov zlMQcGL?uPN*Q?(ruexDZJ|cauS=ghlTSk}U71?&8bw3O18OKIq=Hkl_l8>P-l#aqG zl8-h0YNpAHUfL(`8f-Y|K@2L54z4qAD}wzgwkb)?tlpArR2FT}B;xi5Ppn&rYw&)* z;uXZtw?zW6bOidEAqg6`7+Jc->sM^~Pq2j;vcIS+NT??Al3OLq|H4r#qL7N%ch+;p z-Rd&Icf)h3syc&f!7{^Bk0G-4g>gX6riEKj=5o5%gTX@P@pNOTynjYJe}uoZ5IHNS zKbOE=a9#*oAV%X|O8tQ=O5B)GxY^!7!ILV6GoH1vc!z4>^Wtxx;86C!(aqold6p6d zgk9BI|9fqMW|4j*AOF~XC~o?(vj1KKf+c$g#*>qscS>iYWQpY zi9V?5Z`Kq_`i?VJW^!Ll2VnQfzoj}aR4enj*&WT9J+{T z0Bqg^yy7HvLD@!@vG`sZ?h=k^zqOnIg?=yjQb8Id5Fu94=iPrQA zj=5CvpdA(cdg*f8L4H~;)~{toT>U62Ejuxqlg-&Wu{tH@FlnDzd~!~nQZ42hc&Ky| z*B}B*P!JoE#}ZqgJFni>#&jk2+7Gn*d_F*e%Fn?42vQaFfn+v)-dt(2L=d0!anteY zh2YLqpRj)Jc?e5|KeIZg?SMl4GlCblbE}@dl;#&Oz$9tw2I!Ne$3O{pO{=vco1zQH@?QZj(?>CW-2YaXa%B!#tYfGPg@@EC_jUIC=N z?X-~^djJOmtJIjI4Tf5Hsl#J7RlY{o&e|9e5`00z!)fZQ9Y{0|WMbl2T|+O-clLW7^u{TbsD)8Ehk zKESfi1i4Cp89F$BqFo*w-82`7R%dY{( zi?fmQwFUYDpqlo-)JF2XYU^Jb*VIWhudNkP`ga0Sd&;iVwD!N)=>Xbs2$c%e|`ED_!$j%6gE%bivQCa{s$@l-C;i7krAH)N~OwAJSaXZ zkTpMm1_X2e{pt>wg_{%HGTjBTsPLY5Zrp=#pLbe6rJb*NcSwANf&Xt8ASw5;8TR)q zsA&&~8q>=f`Y&d6sspBh2&N?f3FPe2xEExk_Fr}aD$H{E+W|{``CE2O6-?rKfRh=4 z-WE_040O(k%?msLqJjTFklyzH!Xf>aBCHwf!Jfi!#+fkh$seoX`r@~C4>JAVrUExr zadp6M8?QnYfHys)zHXBP4CVhmQ|<*p-v4IVcV|9L&YsX}k>{Oox4Q*CbB}@E8T^k^ z1ipRWcmilpe8!&O3_80Rg-v+<{R8iyi@!gHPS61zYbR&q0QX;ihX2a9{^M_9zM+Ru zkV8VaF(u=O=3kbJ?RNFF=U3h4^q1=;?wWLJy?vV#>lf}v57UZf5}dEf)kwZ2Wbld< zKraa_&!AYOe7u)*`uzLIy($;sHT~RJg&iLl>jR0PA2VL#;&9pbO5YLN@@kdZw8f4_ zymX5R>U>sJd>qZkNZf}tfDT=y=IfYK9=ahqRRGjc;SoI}o&vxAAUJ+VT*Z!$$=UyT z+S4PrBsi4cLXn3(UeO-$YZv(8L$^0KYg#>rRX^LT@}a15bl2ra6*l$m?Z?}I6+;h9 z&*}>>12SA2sjo5aK#xHy60cx$c#4p-ot5E6PI0QAAxl}W6p1{)&~_)CY5HWmT}w zGqnXfoZlO{-iah*7;)Fc^)XyZp#Gsf)YtKe=q|RCX7ROW^%0}>|Lp~&^%hJE8lP4g zmp#Y8hyW?T+++&M>Qz!!t#{X&Bo|Qd{ub4hJeKiK5;1kATvL6^lNQ4BkOGT zxByiGMH|UR=T#sCft$h`&p(TUM&|yK4Ue+>CAs$K2oQ+Peac)H%6se73VZ|3i{P8l zSr@g4Dv@!a>1+`f|3}0Fd+$AwH|p*jP6(lvw~es zEUAkuQ(*LL&aY#|_Q8ym7e9$f#RPn|=%8)m#T*LHSjN+C4jLQ?9$V) z#kg2e9GH{QRl`xALqE>L}oH;Zd{7>2=|VLZyxNk@__Y`#7f z#h_ecdJDbkzV3C?XA+!NsnhncLG?V18XLD>xqEREY-WS^p!tBt`V$$NZsNePwy5?4 z@3XcK+1p2n7b|7^8vDD3()#vMU`~}rj*l(%e4fiuOFW&of#n#AiBTh-QXl`AXfB#5 zF@otA55qnc8~RVeOaGNwYkf|~Lo&~IWcJf(Mf{_KtGJ7&QN?GHt*6rya|!V?_M%saAWg1)bA9p{t1RCd3-9`AE| zm*w%nzmNp13rlPWzzz~gs;L!!o=5z2BeWsoKV*~|Z4?**r8s9{Emm6zAgZ>jl6IX3 zM9qKWb^ZvM4^iC0DB{ThN5#F-81xB#Z|bROe2;t(g+b>|x!jqBT%~5uJPtZ1nc(K+ zl0+FIH1MEFVw8^>&T+*;LB0)CU2LjD-yYFE)l(*tvs2GteLtR%uKjGniN=rix%phm z{0GD!+84&Ix?8WRacBCKK2Yz5`WJ~E4t~m9ZML+k{FGrYFmR2`JxH0aCY^Q^AHOWw zL)IE1RESJ)j5kanuu2*&ZJkVW-20S?i!1RuN9psn;jh*4uy@7S><2a04zzVzf_Mah0Kq4jU zk-MJ92)85o8Nml{-(jt3fl*N1Ap@)G?YIR^8Rj{yWxvc_92zJb5<0vE-918}Yu$cx z$3O+6WmUQM@#A31SBwQTUsyELw|pfjMAt4$!)nCf$#2Vb8{$f6O;t+B5y>HD^(+(T z^+hdwj)c0{__?d6u?ba#S$QH1=VKulZx@G_{v zD~+4_WSAk+^F78`$icAQ3@eBzOhG9Yh(f=h$JZ2#IBW_$It88Kynun+a7OP8P9enAnnA`c%z zeI0Yc3ea9y7=oC^UBh#0oNA|K^LWIan#1-mf3g zze|yQGK#N}6kqvFwkI_Bltpg*u)H{As}|XRka*RM`)&g4o$IIz@9^zDy_k;0x?iVm1uV1BXUuD#;?UjF2p>_v1!C4GUidlSTJ10xS?;{|Uht%Og%BTyj;{A=M~(*_eafu*^5{f zkecza@8?_1_E>W|TJ*VF!{0A17d+HI2-%*8#*9DD{Irg1va+B0>jgw_>gMPCGxo0) zvBOy^+y%^ozh;kV>JRD11+rDdmc{wD$eb~5?|&0KD*KGUFp9U4m+Ds% zgihst&ybfLEBBEAY{qrsu=1EhjsFVLL-E6*BS(|c)T%lBrDp+y@3bSvu^-{DAq>uy z=~=&ky_>!?cPAy`+bl%kTc7-(n=K0N=D=LhXE||6J>GaihIXb&o-0RlmbAKAMJ)Qk z+@eGLh?+k}C;U88g87{lD8Ykosg}IBv-@FD6-Q{E0%pXIRvuW8pP#vTIY9bP4+jM3 z%k5%*a9j2_`cX(3zL6obL~zGoLby_E&HaA+Ps>n!OLLe;sT8gtch{;iTig@bJ8B}` zmg63DXi1$3LXDjjk>R$L^X)<9Zbsxcrm|R+94=QN4NkibDl5e@{ zB7V|zum{;-o{(^#wWUKHgv? zy1c;5mQtv@&+6AsQWDp8QAJQI!Qv>QjxZ(W{YcD^qzxl}(cBY6Iox8vqFz{1_l7e0 zT_AL{JpHU`B&@kLSbzhQ+n0zPbL}3B&82A?L2Qw0SB`{$2yLHyXws1eQ@BctZYQl|qTX-Nwq$Zx&MU2zK7cKJ?;uo?WD>Fu}v6N`(MABW8nMaS}<4iQ4uH`d3!5`Iw5^(D`(85m^iq*W#<6$ ziwbg%Fmmp4B?iU{msR0m~+Yfh4;zLH}J&;#1_*`TgE3m5gGO~9b=qE;xvlg>0#2SWZ4Ixz6K=L6No?&gJ@Lg$j zg6(2JVtBgZJsr6M4ugZ$7byh1Ww}hBXeR?Q;zNivVe6MDYzvQIM?bSM`AsF!;8o{N zkpd*DVIiW&%8qCx4?Rr^u|lJ$>hWclh-!+T1p{PkL?R-=MOgdKxt>igg0L@dB~e#8 zR)wknX#8!i+*9g{u4TQ4$b8{k0{T?YdClPsc>lSB^9GJ}I5}i(g?k>r@mg^*hJon6 zU#JYro?EhV`042bxI=fYjH~b>)nsEes=(37-V=cB)wb3SyNe0_rFH_*i1T=(Jj8xTd}l1D38^cAd35e;8{z|N!Y=-Gn-+oW zr<3=9xnhl7tSL2r`m&QO)~xA{o9426L=b>zqC#6a0yV}LP9MY`r&{Z!*7aG-Z|_1| zDYQjCLtX!&i+4P3lKUlmrAG8?u4GWaS&7>6#PHsPf;fFEErsQV3@JzF1)`H^W+0y9 zgD->~WUmVQYA$eTx<<)0{>=3V2nu?FIl0fBOU@DmKMe#JwxYJXR!5jW26+Ek+Z+9C zYQwv5MPlHlH8Ro1j#TYuzvQ(zh5ZSVo4yiC6a$Rz^!QMhlL6kSP;p>y-3$Z|;W1_ZI@FSgF5PKJELA>{Y=-WS8|V2FUa6*o3Df-B6;M{avvZUWRNI*pWl_$!hrOO1tXvpH3(sw4w z+%pwPd#7@>%e+EyVoTqnz>7+T2tN! zKugFnT{)N$o_D1V29nkQ>f=+tqo+M;I7aJuCz%YUZNK9mKNf!NR;I1tjD7sxSr+k& zY=bwRCgxt7Pa0~DX@nACr#wW`G5sAE9)Z#+@D6=`AF1Z{$*KA?JjvAQJSGlsVwhHq z+Rdxp%&SJ+zmlN7(I$ccO z@W2Hk0djW9EZLi|fsdjE=+DG_$1zYPyIg<0CN5@CW@6~PDSg8lb-PE?#&RwEz>Y>x zIG(cMdh|(tjmw+Tnq~LULR6*u4NedPwl7e=YQ*ua|66%fIa-A;3ELIUI&4s#8F++4 zZ@Tlu?q^TH;7WMh{Sc7r)ESyf)b7?LXb!31CvJ|f!5 z3Lw`G{cZ|Kp-r3w!13_X2s1EmyGLv99YbtWDcIW8Qp_3gI40`Ag{NzjsEjHZUc7#sxa* zrW*xu=h1)bBViBYX5+5O>-u>NaT9uW&tOwT3tu>*{bq026aHL@MHtWi8|SwUVN&g# z#vFL{MzcvsFudDZGe;GvvO;R#Z3t})17^DvzlV*aJJy7VDqBqX{y9cc+>LAMJiS=~gfCJt3u=sICcLy78i_@Jai% z#U=xAg@W4QSK@QJEVsoIDie3V(jjDoAqT%*w1+5v{Dn4vG{)kbK#~V8J0LP1pM*Fa z2IFN1bBSs(NAr*AyUDpphq>V^SASy8L3W@-2+dWkRaa5|Svm%uXMckttFG}daP+kfOu0F<4d+%70ued-~QgnC)gTjI)vgI8^PP2XQ5QZ{4U~I z@1Vi`+gm7#$ZMO=wK!fMD;NE+pVP^VTqxYkFDDIB)iHKviT(q76-4sl<=)eNQ}3M( zwR#(8r!nd4&HjhwD-Wzozrs8Ikw7))fHP=j3K|n~={h)`S5vCL^eV7C(#Yd2epoiy z4Lo==I2oE$Jg3SOj1hIqSjZf8h^)ILD2jo{i1gr3j%YtgA-C+xTi>FDf6N;(KHTrs zkKJnip63KJlQ4VjykUV-4T@vvCja==J4`@M9Ah3MId9Y~DMdESYOGYrKU^4nb7<8t zvD8_HdE>;FAV9UYHbDb#I|Cy8xw~SnZd^)Gsl>G|=-nF|UJ##RQiDycWZ-0hq$NfK zp7Kqd8y`Jabh_$yn>Un{U^xIWpc6t78Pb`DbxrY^R1(RM8 zpD>n4hO7}uLpUhxIA!FVzBTJDIJ1B?C#bl4I-db^k`UFl#~R9sM!?^X7bTuTv_E0) z6WD`~yBRqA)`FB&Y)}bzPTeTXpj7OQHbEL68F^!;3Yg{k-grTOBK$Bq^_`{Rny zoThs`M_ZD%DXX(tR+OMPaI%fNVgZPz_iY^8)=9}WR5C6t#K)!MVg}v1>@u5d+oc)= zWVW?-mz~;6De52&b|XX!>-2oX-S*nzjZlbZ=}7)@%0AO7)ORc8P5F!Ls_ul=Y2$dR zyDs**&%qUIUx8HMRTL_=wMwd88iS=Y)r%dvtUL{P$q&|>Vit!?IYCk4yk8?NBbAq*!>eJ}+eTbSs`dlED^@8T2ESO%$D?b0PIBL6T z7ugZPfmHE2i*-UK>578#WHoMcQ2IExM^4|~eH{A*Z{cp{hn6q4%fCVzbY$6emy7-^ zLc)c}6f)c9P54r#h?68=R|zJyJOoxUUCp%5Oy~bBo?)Xn@_mi)j`GpZBP##5tUNQg z*997?-0P9^7`<$$x8#Ueyq{Y0bP>`UUUb0lTCwLN!%VmJOLZNiPpDcsv-h=%{7D_s z2Lhhdsy~ic|2<@?pQc`vlZ}G$WI`+a&8XkqEqWm>voeM%F?Imwwch zm^PXEy7@yBX-35Wm+gNBMXl{<|1BoE@9Wb>L~j3M+g-*G$}{G2gzV@cCKJR8hvw&Y z`ntP``#cb4CnCvl?3Qz>8(bCWMzWLBrOEgSlTf@6`R?X%smSH+))$*?#H%UJ_)#2# zvh&w@qDT@~`P!@cNm^N#UFAND08YOA`1f&~83YoKnl5Ba1bSJ6Pv}cDkA~vZkX7j$ zh?^Q+vxm-89=m)K@2kTV#~9`IvLLYU_czL(0U#~n!CnNJ%F^b zKL3F&*vMM;+hvFj{-uPVmF!1Dod-{+$m;1?c1@=@F}O=U7B$^m6qxKTQLL?W2oW3M zD6vKh)m!nE(D@^7>Aj3Tj<2&QY?B~Gq3)gYou#;7v%5sfgb`LCd|;;Uq>d7u1;}8#}0$DN2_}FbRV|wH%7I?x_n-djtW6EDpw6#;!`s<9XJcH-4DJr&NP%?&M1~ z{u^lxSB8GmW8UV0^$(W$_m^1~%;oR_-f;!Jx`qN}sIqwhS7o(Ck=3_Y<{TEomRv za_HbK^XC{zm(^1w;1K(PoBV_@qPY^9b$_UXOrW-*cua}uI2m+G+5qwCeZHGh+8s>C-OB(efblr0pC27uNq6z&{0x*v-Rcmhsc`r#E^kw< zhG!`D#??4UqzT?`Z-|R#T=&-o$NHlH2TX}uhhm*#pJal$rq164NiHWmW6F|>5 z8Sibw9nChTi7&hk&MZj0;Q_mrG{f_^%+b3Qg8SwDra4~Dqt?TXXU4yTbDFqqT>Q|F z*b+i=B=#31Mi4zU;dLQ?YVfXX7nAcE?vp2Kz1Ewut@51poo&Fio%W$KPYts&?iIe&I{I+&OA!WlzwlnD`h_R8D_m zKSgl*(Xgx3c@sBS7-d;a;Lqnp4Jd5I?dS1pq6!_F3 zf5JJn4J2=lR~_4yq;BzI&x`iFBjih=a|?`k-etGdCUh|L1EWNwrG?%rmp0P69ZkQv z&*p`fIslQN|C4>fbaBd}nj<>1aEo8LABY*7MwXwdmP?1}?64^RG0^Q-wd!|VNLFuW z%t_Z6gRnE9uf+zDyaUGDwNZ^F+zbrl7HHv`o_!*HOTM!AKoSPOPe5be=CEw#D^KV5 z;`!ko_910_$z1WS!;Eu1)2Q1zu@4lOQ7tdhFP@j*mRa^~+B7zeY-?}GQpqQ&R30|h zK`lq@&LWJ1B{d%>ri+Ef&nFx)96STICYCLXhmWe4tg@}6B8n0(w;laxbbGSrTKB`gGV3{dKv_a?0*CKH*$j`Dqtk(457Ft@8_&dyAK z_B0eG=qj;KdLBhGRN|!WqSdW9R?jaiwPrqHh-exn?@jxXAE6LoPp6hfSSLUEBz@{M zOa-Rje$8HksqR&JQ70Q*$XO+hc&eC#FuI%Qn$gQTD z7TaManF>v7bAf*ld;Z(35)&R7V=SI0=kVLRTNH0%sSm@bE0Kv&=x%GsuqR*Ap4t~{ zyo%+Up=;9=FA%So+;tit>SXLVA$`L31uA?oi4Z03)nuGJ?azGXIP1d6&EN?6sSpxX zCgybrQmbm&O($dDOfJc0syaEZ zlFB+06O@X0Aa+J=I zI@;QPP>{`T_bHVMm2j$jpPWp(?Y%wdlQMDxC^b81dqy7G*G5;J4sUw4FoWYtysHzZ zm$bk)6N4<2XY@+DJkqrN8x!a*ameY32(NaZ1(H+#{G=zbSmolouJt%Vv8d(&IWJUo z2O$ru?z_4|d}uZ~P2NOqSjr^6Ps*;4fQUMdUPqvmo`EtT$h`rc|^g9B&g?-`9 zDXxqLjW!8bFP4e;!o$hR!9ss|(#fb(4QMe3zJJ7Q$b7a2osi`60|BBGvc?+jE8qIP z?qD1g2!!2&1WeqK)$GJyIw(q_1Ve7*xaFQxo#9IyZn8@s@LjI^9)v1*#jW6(Hr< zj^~q&K}#G4Od*k9_NWrp<*18{{3B?~uzZE_0mfY_{3)K>@UuwH+@7zE

xn>y~m3 zoi$wW($||$Hld};`RJMuix=+=2$S?(be_vsH0In*!=!y)L zI2A8SSGUz9RkhM}bScHMF+0Kk6xG9Aimu#dX@2@4PV`r24lRYNx-4vb1TBA(2oO%f zZBRvz?KZ5iF2_hR=MQ^{>-po`lF6mx99v-z+o$jxA5l?1klV#b`P}l+!k&dpO`YfE zLG=KjM;a3)`tG78zL;1HdUCyTb`}>4zX2xq4q$H;+-?b491^Gd7~L!l*5OUt$JSta zzo`5w2lqJN&EIQj(Ot{$>?+-e_6VGxc4M-`I;{b1ss}LzGnGQml&>x(KyttZb-t|M zhT}pa?%U95KRt|mpKJVSb`?I|uQROh3yK1ysFZq|v}XR?kW)88xvzpqTK0Z8TM95G z4)c?Zg^fqkxQXl4BN;|+qSZIz>3r-&BaNL3A%K zj=mVb>}c(D{{9Q2a^&c#rg1ikJGEWg@JeNsWhSYPzD5gS$Vt4UiPHl)nAg;NpTu$d zhWmV81HAVXwU@*<#zG&NVZ(Kzg)^GjoU4)A(0*>4(^k-{J)6_Y`#xjHsG7z4(nKl1 zsDLBQdPwxUUV##mabx($3c?cI*xwxjGlIF>#HEtz4s+ro(`6eNfHLWgLrfk z(lHxxR0l9P3ayp>j;A1Z3Z)$|cZPMK$oIQOg*YinvF(UP?<^`^SG_yZT{c#2_>tuH zHx7+!Dpe+_M+7_RH#(G7RBV%p0zJ0+-4{AZy&E=8BF&g6sKlAz>o)+wreb@`8@!51 z;2~(G9eM|Yb=CqVN`nf&j;;3c$vjT^TtY;@xiKxFf4~o1v1aCe*w(&8!LQk+yCsPa zCUWm*m(3Sn(Jb_1N@UnFMdRK`tZjkJ`Esz)Q*QmpfTPl0(N?fc&|adJ^YFsh+qGn# zJ10;vw0WcO-89g(Hod}IlqJ%yhW*L^QXwd#)Fi`BZnHytTb>z4-LDcdoG$!l$Cham zMR1@~o&ew9{cl*>-Y1FYF-(W8wvZf~LvDhMaRA=sj&^z!Sm}5h0IUc-N*=+}FHJ{) zrr!Csx*S+;LjWxF19z6j_2CY0!+bQ9dy^scqy@>{<68JB&1y}Z*+pT`a310)3QLm# zq)S_X1!~Ie8{pK;?5{jsdqPjv+3(lrta=TUp42FZ7cDN|4HZ)pb8M^_0$B52R9x~s z*Xh3NDq~6OF!%o)y3C2Enc5|%>5{`e-!pp1P*JWRW`S5u&uGe=J1w}cw0z-C>$Lod zADdrj%YiG2 zYvYf)tzOk8;YZBf>&Wz2%4fZLf!PrVhd+Skex|i>jj>Udgo3UN^ z3l)oBl{~{ajQ5W)teLx)7gl}+?LL)5&k zn{9;%UBp(b)*{uBH34NC15MSjop87MTwA-yA2NzZ6JG z%cvfH*sHgTr3_f2iznKw`;i!w*ZVA{@a=l(+nLm>i|_VRwSK?GpFdrabZlJF9h#gM zjd*S3)1^t4czx-{GP3RPh%CrSoyvc=RpoZTG4ke9Ye$ioNRzV zH2Rqox&10y{vFk54AOiKEC&dS=AncZzcja`pev}iyhhOB`6l#|vilv@z>jv_%rLJLG$lra*(gXD6ivb#jXAgVF8@J!8t3U+lt^6Uhvwq=y!pO&=i4)zhisEpd_6k!>^ zSPB&vuSJ6fb}3|PuC_G%N_1$kXXiEghTl{l4<*)d;Ch^?bF`E7yS1)q@-1+e%u)7Gj`|lAf|?-nt4eLo`u#E` z$?4)KQXpfScfyRqlqX-ow-JH!ou`~1UyHcfXGs0Y&v{VG+N29?R%}+3byDq=a=I^z zITpWb3s`30rX`DLbER1X6)#`@`tfmDm0iC44-Y{5+73O2m{hBbKu>L={Ucgs9|{FU zn2$Q4{GytD>56d@I}qeu@)ehp&In}*Vt%#(=9Ec2$`dpFL(hfu%}dqRY`*W3D`S3vc)|v36{lIrI zKaU-edGKcmuZ|?)$?;_PHbG}W{Y_}$ilSDzL8$XqLxm>E!tsw5W2)Im!QV?fw9?lH z%x3FQqwS@LR>0RokUvD9niS;Gw36WU5bjt zCc}AR9`vnu&0Rj&=__5;>eiP{re5sn$pFu6ejI$GU__3`P`Pj-jqd}=!XxApD z@{PBnKl4YSU4re4dWYn>o3f&hL8D3Qa&%=7>%Jr15}N@h?%=j#Me2wm)t}wGY!!1o z>3lk9>EDJb6co+ck54cyzT%ZJ7{90C+!}Dg0WKfzclsyW=Bnf3##cKe$F{+*E0^yY zR@F07GW%aD%d(v4ntq0v$^l!q5}K3(%a~;yfW|pc+XhdWf+s6Xi8=c)?_k9qI`(T% zXgPEyD3Tb(8uGw1BMl531vs|GAGq1M zFvTETsb&&9J=iT#XjN*1d`i5RKsSx4hfH0Q2v2>=Kh!DIKf1Cur`A8dl5@xoiprf4 z>C|VudXe3fZz%fdBu~eC6n2CurmZ=Mdehi?o6UL7pg^~h)?SgGlo7Sgt~K%%EthEp ziI~W(J6%-ZcN!N&!VgX;i>l{eyvqMZrIx z!Ct5>ahRj!+tlPb(@WdZtp zTf3j4&Nj`>I(VC*?A#mV(xek>{XR>bWfXgMD+y6r(4~<|4Ukz=!$<`{-uwgyecr5{ z=A|_O6qwP_39la8Sy1^60Fb7I>})^NvBYQSIaf_?LBsbyddu5#G3XFV^SN^6eg^Sh zaus?T+m*R`)A&)P!`iYN(CeSj<;BgBE()p-#?@H+@dyGwp7GM|4K!hq+o{MR;gUsO z6I5uo_fDG$E-n#CL+BotWhO=Z;YB(x7n-%B!m0JlO`Y}R021%Un6zgC51{I%+$Lb{ zB>igbCKw+kJ&yW$VJdGoTk+Dh|8k>qT#x!jJmqeXEo9DD{AZP)bZ4#F?B>Xhs{UCK z2bn%pulM;jsAJ0=@9-3a<>mQ!*3r><0W_zcvup#GuuW=)x;ddHACZE<9r5b9i4djM zFmBf%Q^Dt#ATYB!KX>zc$hM(koz6Hnm)mx z=_R3uDpyH`2`maKzn~|hH~-8j)o+k7&-KF8<5y@MlGTpql=qZr*(s$L?b(T}^FFR( z@3+)hQq$7ZT{rU9zK=xgFUIzO9f_{FMSvIzN3X>|*9-zL0LLe=5Ct&e?O#m>1a5+0hw|Zx^YlN~jJV~_RG04A&^~R7pp75BqvXBldZ`V@GN#KuRq00c zcRfVL)*#vs4gRu+ImDFXPUz!Y8dZR*}Su^8;REcTIXO){N3`%qHOu zbKfoVYgqMwL=JEI4^7_KKGk7t_oCjN@eVjy8jZ`K3(5!~nq5X(ge0P; zMNcICmjz!_yR1l=_#>Q=I@S7yv5bPu=2A=oT+@fLrKM-{(DLIk#Oj{Fr{PA|RtsdJ zZ1iq2-b+Q}mvehIxTsSzW-%q*0~M%(a{wF17YcZC8qhR%LNpGb{DL`gZVWk7hF#De zL$dndMWlOn&qzLA@uL^0jF3+t-s3qREjQN9HM}lbR;Vu;#Th3Gv^c`THo}zK;Qmce zBRv1MhW1H`QgvZ_eAex_30}?O1ZO`>Ebsn&TPB0l$?#m#DY|hSO=a;IwGY5ZXq+sW z&hU+HTvVQ6i5mud1>?}(3tlZfJaX963@{77TUqQSPp!m!HM#m%TG3)PKA-H#U0-^C za^5Ys`EmR)bHlJ~_5I=lc1I@TNz{@-UvVM(^cbIx9BxNv#NI{U|tUOJoWSsJtxm5gmlPl#- z@=O&FWzBKw?>||Ri($~`{!%SID*Uk|P7aY)Ldx0kj+)hL@2e?sy}wY+${p z=1bLJil54)&^5wRhMT71n( zpEz6&e;HFijr%>e_1fS-T~`6_9ZsYH7%Wb9N+s?cJ07bV`d57j&Lecgv@A68F^p?V zyCmrCk4H~}h5@h5rPTg>KIh(4pPm8FHckaWFR3{FIl*ou0%Z^74FLJsGXQ|cvR^)9 Ic=hc60bkA=;s5{u literal 0 HcmV?d00001 diff --git a/assets/rancher-monitoring/rancher-monitoring-crd-9.4.204-rc09.tgz b/assets/rancher-monitoring/rancher-monitoring-crd-9.4.204-rc09.tgz new file mode 100755 index 0000000000000000000000000000000000000000..1acff385cd1807d9765253892add7b44ccee50da GIT binary patch literal 114200 zcmYhiV{|6n6Yd>b6K7&iY-?iOv2Dy9+qP}nw(VqMTNB%U^MB5J);j%RcU7;ediCyJ z-Mx4HY7<7lpo9Ex`=SD&F_2JXG?I{E2e@&u8M3G{8Y{8=(NyAMms3?`mr=6>8dw{< zDJ$6UNtjq#ft+_)IN-I#{c6v7ph3ag&TUNDEM$#&xP88a)jWuFqg6nNFW~C-keke} znAucU1vLR#G>+l>`ha5gb9Y<@U$m5}F9DJ+6_~1kNsytw$clsGd(~4bzF6SyiT=z+26jd z+w^$dF&r2p8$I#2u`7Q}j3tb$X>SOO3J1jo|D+G%_Y#y4L6kW#{Hb7s%6@puX*hn( zWp+bl*EsjQ7C*v-R&vl=nu*WaO=3RoC`Rfmbv;5od(y;CYxOQ$7pO2r@%UzliVN}P zZ~9KElrC*2rCOsm2oMi9gZlKMYRr?&57S`2ef+1L(58&&Tg<>6!gv;$g_>>x8v|m* zpz1y7ljF_e<#0PCB1-x7x#wVWl3Q~4USl55(nSe17{dWh^`SPVfBh&9ro~G2o|EK+Ck zrls5;o5cu+bixr;V-e28;DV`h(f@^*G4OWEk~j$D5tm04|7Z;x9pA zyBt{*TbOb7idtB7GVNb9%=Mm(qJ>Jho1q?p^*^dV5PY&$dSt0*e)OQ(Ptz9Ah!T}m zNkHPgE_hk^=-+WgOj2>R#Nz|wv)4!d`g&ZqVL$Bgc{5%3;-6Al%rIRLnb==E z!+O#lEIIHQTa8G>`KcsaRT(rsyZtFvY>wZWPet1E$sLAX5Zs29>2J2(F4%>*6_E3N zue*uJ@>r71(QnL?*?xNpjs|N=jrPTppC~iLq$q9?X2t`6EW4#E_`5x~mU-rr8$F4D zc$}!qmY(cET7zI2=Co^bFI%Y{@htHlMobZU$C0AW@;*?uU@Cx>5-i-uvg#&>uE1x7 z-JUefo6s!LxXGY4AP!ZLKmBj~5u1E%y9RHXAcL4DqqdliTKn4 zm7ABn{hoJBVeJ!%Qc{@QhP$0pvgq-iO<*pkb;+^a60soQy9aAStrLXZS;_XiFAvM} zRRXx}*RJJ}u3400Y?gaeZJBzK4pph9{mNSr#_5Vba#X$3WA|~eFzE8$0~RlqoAfB| zUQ;GTM&i&Nx;dkjEn9r2)5N4T4yTd;w~1XX9VHb;YCkl*bv*h`Sy?}c;va@hcM~*I zmltT|DiYjwdm1MvT{AsVww1CE#s_Mb->W(f4O&`5<9V&oz>5#AL~MVs7>r&?->VHK zsS>fp6+6ANy_*(+EKkiR`*!I?} zB6YvEd$KIozPi0XYzjV~sIn1zaHPa+`0}8~#FHFQNr>HSwKC5V-|_!0Zhx{9_e9ti zr82g}O=5F>_>7{T3moRCR9;Lolx*Jxo}3_uW&W_24|032`GV6&)?MURQKrRy(Gi=v z4Sl~fgCF#(=zi#(vNu9K%n2h*#DP#Ypxbs`G7OSKo=U4MPp+Kqy81lf`vQs;95sD; zdfoopZax>V5&PPvbSpn1Y)=1YN`>(Sc{z3Ran<)z%O0w+aCq= z2#+9c0Fvd=hq(fG?d$i=+4a{#Fy)t9==jp^R~aHq&xo#rpBm@KtK4Qk{S?$FODHan}gxLmpS}iVn%ayx`)8q{#@jCi~0BP3rXND+LDQHhmRavrN z1YBDZdz?dD=J?muHTJ zgA&s(ylx_q1{c*orCrchnPF`2eAD%TMF8yd#CLPt1L990WMsHdoV}ftF4j+BDMGn1 zmu>I2hZY8yGz(r153nOA_6|lAh2I}^3W`e36bH5P=13n5>7wCj0}NukI5}qCng=}@ z5e;`wSo5q~BiLTA+h;XBtC8Q2jQRD8f9y_0t8qUK8G$(N@MMs-jM$SVo_U#;qQS|C z_3%h{{rO&i?x?3&V%=vZkZS+(F7H%-`NhZU_UC7Z^_Sb-zYd#`yDzHjn$fvbKn?*R z%`2mhZNsEln<;gCKchi$93X~;T*)+A@jslJUHd;*$N3FK+*pmi(7>X1%@m?N`-MeBbx>zD5!KpCMA;+ zqfj13CSV)T1GPT-T$k2gxjD2*A{kP3@YlC*MOUpj$Si{H;^<0XzNQaV*onq2H zOJeHXwRg8OxvSPY2xG}=^ZL4>W6(pxI7;QkFDQ59c|g@l%%1Kj^bMbxWTkYSw)ouN zo|ziM87@-{mVs^Wp=$wx0&9qYvBvZfU+W#hJ!UlB<=~uto&1hcn2Oy z*tIY8MB8H4S!tNo$U=1{lTgDrZJ1AKPz|GJa#H<{%8tezbJ%o!?86^7mB*lKGl(0* z;3>w*=^cc;&ieDyN9Fw5z@!+r1eK6zQMFSGI8^HSj&P)xzx-)jXYXL=p|+B%4}Z!D zZJCmDf_j6gpgoz^AC*Cq1|~wW2e3ETAynu;igM1H5t4x*FFPkPN@*4_TS)p$nobetW8p$tS0nFpqLV+63Csku)hdKIZ(8n;8t8ffkL zym<|QR<}L=CfLc=Q*A-=ZSum}c;246>KPZ8vkyNlk;P4554U(IzN37o(Jw0_wKx#)yrj{2bP@gLT1*^Z!vM0hHsu4N?tFIst%! zlXF%}Qa4fAS^3mi)a_i99jo(^!e7Um3p-}7N%AswtN-*8c}|ouc*BWGPTEx*2Ji^| ziFd_RD)tdVjLr^1{|t&(+a8On@~9taTUlo)vb1zAEalEjS$wsLw^ z9{WdhS1y=xL)SKS0^8{&kkJ!v(Cv^bUjC;_g%xx6W_r67np9GKFwPT{_aFCC6nMz1_z<^o5C?pbX=rDy4rEcw;u zw8Kg5Z$_KCuBXZ8)#iEyN)Dg{Sf8syRFsQ3(d$~Hvu4~WFa(bHNjD zE@LYqTh6mnv+pr41GY(&!jid{n8tuWK_%1p&;OdQ-}Pcd@{J`3jA!qG&ebwJNHFYM z5o#=}_m!pTmM+i~B=jIK#eJRGo%~!U#k90HklwE~+Ymd2_eUH8ANP!9>Vw?6LL1Fm z7H=-4HVx~v({k5xob+m`r53|ibZSCpdzp(G8vm>%#u4(0P1g>nXm}-#p#)*5a4nS zDZ~VsEf|UOH9z4ztOTcsXT1EfaO5qhtb2&8AO2jyJh+BW9M~-WtF|r zNz+%WWZ~rqFw7IUV8(nIS}I;(N=6wLS^DQ6w7zVUekQrzJi1kr_yHCbwix%i5md6u zpJc%5vfim$R9v}HqW3t&hP2}7A!`(Q*ZpLtI|J}vkwVbfrHu`2mT~W_R#6lAUeKe;pUHHfZnJJOZKmzu4{xinRaG!13(sLHf6FDMP_S0L=#2 z>KYsJ!!Uj`T27P!or^PWHIHAseVsiXZ$;T+EbPC(V9?@Po(DUMc!5GivMOaakwUj| zBpM0e&+CH6eb%U0Au#?t*oXeQ^y6JfP9VvIn2#7W*eO16b?ol2r_NePBn%OaH$g+4 zZj04YYRSqGBH(cXCd%gaJ5^>O?V0<1pS;lle{DG#nCX2lriu4XgQKr1gQ^|pB9s0N zrFPz@TX$UQ@GA0FusNGg{<17W`7FipMyua8LWVaIa*`2bl|%fdtQEc!KxRb)n)?f=%7jevk~ZpQ9n4_n zjT^iaj7Xe~Y_`Tnrg@TkhG$i}tKH(0FXhkPfajy?Nkaox(4#?V4-E>7<UCiki>ct|}a1>JszhYB?>0~reW`xQ)DZeR8=JaNBeFX$B^6qwe3U-E5Tp9SEi zCwRcF9ZIuZy3z zD<>e>Mg%-Lwnkq9ddEf~c3<#OejBT$UsS?`9SASrT{?x+p8k`l)ooZ>9mB zj0+S4UIk42*n8N5D{osz(L{LWvD=H-0_HBkKXsh}L}SBGx=n8pF$>JGD0iQY9z~Vga}Y zpPP~0B@-YjOPR53O*>DTUuQ9QEO21|j%< z&-tt>H<_Uaz?YrmA1@0?43+Hv{$rIx*oY2MX_Whvf5XasvxVn9;6%4TUK=G2v{4&X zX4vgmjy2QZZV3g`DT#HVnUNP<(+tlND4-`A0J~+nRb`r*iiaH_{mnd<=&8*UR+n(V z(l+#4OTuwL>hM4tMQ0;-2rPD}(nOYE!%>Caw}~|!`_DY>>!gS>ou)|Anh5gr0*ScB zyd4Ndq-KCPWQ;p)frqqhG6zAkm`JymkM1?#(jI$*FOjNkk13+$fK(;(FDlJ`d-Y8E z`{m(CLq<-`-z~jHSkmOa+L6fc@1Njuk=ne$-L*U#g{ros0Tx&|A2|(&D*Q|+9PK3p z)Rys(Vd8Z}nw#!~Xfm$NUl1R43&B!t5ov17>I5q%6Oo#ryclQYnOi3jmaFn2vShb6 z5uc|H(*9Md7ahorL~TwO43Yer=&DqIV!Zw}TDfq#(r~omzL5J`;~5Ag#(nic)?w+1{jI&fJBXyDYb$PG4g!_Uyg9|f~hoyK0 zAROOQPsJZ{rKEF`btfZwTsP)I5iPB$Ghw>*WUHnQg>s%(7=i8NXdEH2Xf6tPjDhUr zYIB^+g`{xSK`QE!YDx!D36k{Hm)4*5AG^1w)la{gf1h*Z5{uVC+P{Hb2!vbl4?Tjf z`Jk__3Y~!>@`uyaY9X0FD_Ht1FafDm;TTz7LCVa^9<_QF zg+{TpmQ<9eaub#zJZ>A;mY350=gd|=1q#`%-!7`qf%0lD20||^THCd5If%{O>Wr0K zdJNPLK`;5!mHdMkgvY_#Qrcn}`3+w|oC;7c_OGqZ`;ABy=j&PcXwqd;Gs(Z?U$#IKOPA{_d?b2~DzP9EDx6$(RodFc-C< ziyvf1L`VBBQXbuWopbws;siRR<&7CKZrSvGc2(P@`m54??f6Xr>15neS~`IKSX>Fr z28|3$R13Jcdtr#qI(@pxd|x5t;Ju8B;>&4~a$$cP0MrGN;|NXlR|}}d0v8Lq&=K1T zFSwitj)%}Y@!0xv$eL!f>2NqK9xyT2IhWP~5hQ8w`=9!yV-Jp0O>Lb>AQ3fY%-x;* zf+7Pq;rOaR|PFycykZ(}(jx0`BR z=tCYv?q>&qsXplz!WSpfBuq7-P}bv|IX`D-FGgb}n&L zz~TkV?t4MM>+XAXkXOw5UXe4AN>ViN<`oY?tX+svTfJusCW)=hk%zb_tr@jf<5Z1l z#G}y13WBQJAgUk)(F)CW_5fS<1zIzE!&g%*Dh8>*i49{}LZ1Ry*zFaJk(zsH2mMLB zooJ%43$`0?;6u`I-M68bm1qr_0$LMeuz#G{acZrz>4vHpv`o#81g7`}ZHOt{XD<>7 zgtQEyZdNg%X>jW7cvG`N=A6&Ew~v>WQnqTqDVfEj);g6W2j{=)v_s!nj2=Fj=y?h= zbmhX+Ju}01jY46Ahv)$EQWR&%V)`20D5QcSMgZ6q8^2*U0dsn zBb$sYAg1^r!1vBOnRlj%pG3`aXyM0yC?={OfGbzYU2x;je{{gwN4*(L|9j@%^pG@S zmU;!V)OM1v)jDx08vxJ)C7X!9-sU0Z-*v#ODj+)$$rRn`ph_r^{&~O2tdoxjQ&ANJ zRx1L%#7H$(mkL`B!mZwa(es#R?j^YnWKpfX$*gP-rXOsc;VSb4&)LOmZOWn8%l2mCO?+w70~66vBlDQ>7;=FEDqgj@y2iC5hBfwYeHAkZYhwLG{@M_&U08 zqxqp)SDhowTIt+ry^m7{a}ts5g2~pwuOrsN+vT}Up56yh_KvBM{e!FDT{0#}j?qp- zCX1B(GdeM_KcJ>17A5R4L4n|=5^8l{4YsZe!aEhuR{6-tPqOh0Lzo2LQV)_#U;Gkk zE*Y6GY6|@hM@2{NlKFVJyNxVN8#5_VGa7&%0yV=)?lLEpawu<+hStZJ5W!~(8`BDV zef>k<8$z+G&|=H)W(6x|gY&7a=nE|u!PtTL*9_S%jdQ0lE0c630!d-&dw*hJsnzah z{m3iTwBk6^%!awfHu5^G`a_BH&Znb`%Lh0ME^S}`4H7^eKP_E)QFwf}(}2op8n5Mu zs;Q5csdF-D@j0^6Z50gJj>5>^jgQ@^8FT+|R-^Vrb`DaF232g3m$o4f+yQ zLXSf1O*0)4#jHMg)pQF|rTW1b4#UdLt>9}4X<~jQKpjGKcF3!?NyvF>EAZ(JfjUui zksOi>z1AIHOd3u*eDjuQdupctcOE~@obgmI8`ojwz)sK0xvGb$z&wKF>6TsgKzR(r zo&_rJzF9e6Jme6yl{mEqBwsiY!Os{5lzLO(VzyB&R}ayGS-rH&4Kl_Vau>hTs3LMX zSNqE+y@}EcL`mrAJdTpdWF9m5E!d-uwOdIwN-l@c;#otmd_38<*W1PNwya%v5QB8= zL_trB2OB~QYm784bL6J@52{QAoT?L81-lA4$OPku~j{5^y zrLoqc5cpDi@P<Pot+@XKQ=yCNd zydL;^7Fkyt^S7XAc@}d16&n!d7R=0@O6M>XeV(oXN5c&lhSnHi0HznBTIzzKRoYPK zYHXQG)R9YFo`P^YzCy|{QKv`;`Olf_07PJ!tl+2gn&T$f;ULOJQzSC76XJr42Y*4` z_bz7m^wUP%=)t^_AZihDI>p@nTdlYpO%)q_WumHYD6DMYX4|>)tKy=@aM+-aT42^JKn%XLwM}OOx>MbRvWDi-Nv)z@5NrA4cLs`UnNy*; z?XDlZ`frWa#*0KVLZ^E9bP2p=m16mi-C4$;0>0|`wWXMq-&z|bHU_A@ZCP3u)@7-> z$?j~-1{ye$ZtLx|+Tt-{MVT7dJFdwT1Iv0nUwq4YgkQmTwExD5luUN0;Mf(A zIOO_2AE_8RQYivjNA(koLlNC69S%^G4rHBlo!JDD=6L*NV&0f|_@!lvKsp1{aG;%e z&R7^8Gcce2s3M41^(f-yq97{-L;vazmf`;fDOmwptMy>L)lb^G^Bl`(T4nP}hE!>7$$kFS*_sYmDo*al4EEMxQ^W z6hWwK>1~D^ue78TA8C+R?z9}boHWmUc`g!Sq8}6r;SAY3wpipE>%L?Y67-z6($W{ zn#(dx{EJvX^fHqEK3!8!=o$k+>Ug!pnY^tDR-P&%iEX@$A~92M^Yi`kJO=Bk_`qMx zs;HkvMF@gjL}X2G4udrD!2gTQ#fnD{6GhLhMiPFe`<0rH7yKtQLaB0C6x2*4ZB=TG zar?<*N39RgOaa|(I-W`l)AJC)xf(*INUfM$QX2M=y^w`bsm1Oopd$-hM*joM}t2G{wGpsTAed% z>)Mi%J;l;9Y_>yno(26Keh!_zCsk=c0@b^o)~dzNk#YOGe%QJ=0sS`GYY%>DYLBDZ zr!npTjK6~x9p}J+T0>G$V)YvI^w{QGzpZXlzX$3GSjgoGnwO>?z-C@-QvA}v?V64B zQxKHNV0gz2MgUZ=c|0cx8OyK#)~$8hzA1oJ-0 zFp^A6FqKCwesi|=?hI1>G9vgHL|djBFvv+d9K2=Hzd8OK1J$j?tzEUQ5)Zb$X<06| ztG}u29`iC8k=z{S7)kq9wtRzu7!la5wjP8Cuz?BZ6LLMbS(4&brZDO(H&C<-+I-e( zLpy$qx)4%^Tf{k@$c9*sLZenM&yIxj&%b!n-DR-5{1zr}{RMUgB}^;l@g`p+Yx>ZS?_zEs%X zQnAgDNq5I&Vi5c1?P>IAhBwYFi#pWEXa>`*X=Eo3sSWqmcQgDyzxW58(ZLk98;jVM zvhGh(n~lOxgLfsP1DTI+MYGs<(~Xd`hV8bW^M7N%!=lXoJIl9q=|$b{7SbxeHO@ML zNXjAF@tSY;Y8mC-H@@pi&jHf}ffqs^T&cd~|xaik)w|m{2a#`8K+|1z|Js<%NWjD?_IhncC=SSX7qwn>3R@EuY3 zZ((4nD+OHEpg7GK!KZKbge!x(vgGY8!)r-)U=eMqo}154p=G zgTLB7Z9QIk2E@su#r!wGJ#O{U49AQ?C~UMHr$Z-_UEHZ(JgQM$TW#zWVkQ}gW0i+3 zrK;nogjzObxAZ*K>OmR5)?$KoStXYSr3`7(86AmfR?Dkm>kBc5i&Wc&Gb!5leM7DV zCr-Du446^mhTh0bg?>_|ti=)Ks*Z)`T83MPMyn+4ia<6%V4kfKEqhlj52faYx%5u5 z&EvL283hzI0!LILhf#2o6&fHqOrd8IPdzaAE0j7zvdF4*pg}mgcp`$h3MkYNF1M|T zU4~)M9FrYIvmqfx=bi&3mN*pCuOed0doSM0j$w5db9d_@Nlkb`Sp|vI)XDGe?B5=L zXH#RDy+V`U%@pG$`+K+YhA1WNw;A-8vO$Pin_S;d*ecx?6qO-T*Q2V@v$81NW>>?? zG&=Sxrx=rsvONRJZ@&_MN3sm-OcrXyM`5{#I^dE7U*-9g0p#}#(QL8o<~)iei`54W zDe|NFDp$h0-~LJd^H@2dp`yiszI-SObrXq$ZnIt@hYe7fA4^3y+;MNUtWN8{{7Y>X zIw)|&lFS(JqXlJI4zU@vH~4nJwf~s^1aOSE7Nowo602U%tQ!Zs-9O~wB4)UF*Y%4r zk#$R1u_kjjU_U%M0*9XM>3TZ02V{p77{*<$CZ=#7OlOW#<41yngbBYJ1uzZ6;5DowN`dMZfK^q z?BWFSBP>JV+&n-D<45HPP-Giucd?1Irwh(|5eRO9?tp|HaPc&Xu*S-{tyASe<_nMx zED*(ys=VXSyh6p z=#}YIX$Y`l?QyWrE#`ID@H_At@ST|5n6YOT7NA~76YA1iZfI6jL%P}zO}&V>kiyd7_F`_?p7*-1j}>d(8qb>=pKY5&YbiE9k>4h)6JuYgI%qBdi;RbFeo_l4 zXsXV*1}cv*M6l8NU8b=#w;y}~bn65Tc$YU-kR!iiB$snJ`Wm@D>gB;o+dW9NCjVpq zT65V^IY(`Gh(CxQy-_u4)OH}HlnEPvJ`=iL1m~0rJyg%v{ zpkbN51~#wZsB^ctMmsa!bkLrT=dBEAANZqfbzv*=$+H|h)BmA;#-%Wb2hJs0wS=A% zVN#ei89ov@=*e%kMaOSMRuEo1>5B`g0I%aJ%EL6umDjy&^#jx;kYDqgt7AzgI>C^K zp~HtBHz-MvS@DGR;uVr0D;EXEhl)wpqACjRX$$(77_jQfHP*Q|Cq?%r24kB)2-!IG zgLv9SiZr`O+5s|adf}l}ZY}(DdGw8>zJ^(f?_;PwjHG&e8~`WV?!$?nOhrC`6GEle zT^yy?kZG&`?2*LJ@o%L~6m#*t56+nT|MdTllq3892EA#UGucQsA&ha`LcL2>?$PE< zR|QjRF<9{IC6l+0dePwN{7tV+Vu~Z@}9k5B1h+={SX`fb`3po z!L9f}Ej&i-gjzj@L0y}lteiazkEX0{Bv$C(eGOSPko^^Dzp%T&5laJSv1naK60Aka z@)JMmkkQ&V)xt&mjC~s7p-b4d;X~7)!w9=S=ES=kVJ-kt!p%Nh(XEtdXyxbyZ}BzK zShPMF0b3N+>iK(c0yHmeyYgTmpcbW!lqy8#w$+53m3z65#5N*X( zN*SD(j7T`CCy~BxfIV0~4NlUIUKC*`U~l+TS=)#ne}#<$oPJ$UEz9ZO>BnpZ3h4TkhI1yuHXTxCKqES!K8@^&5-S_?!)r$Z=5-4~OKSNSs}VdJvlw zp^#UO@4|fiw5p3Jm|!VVL>V|%ascf)0Z4#354@zJOwWC4P&;+&{j$xa*#pf7r)kCh zcsBrb^=C@`CPK-}?ZpoUTo}}F28h@~Id2l_7hkSaM0Az*L6)>bNsna$q3#xC1lRG+ z3wd}mOJ1(rW1ilN&T998f{1xJxpvoIB?p*oa&3l6w|p}HE5p7z;Axv791K#dGW>0| z1>5B5mKO2mUJ)chUx}A;I4oYNthxxcGL8P%)ecsEd7?OQioma$XeSJ=|6F^W(Tb{_ z8%u|Lf`fAVoj`IMh}9zidep-&^v{sXh$m!;N(Qw;LOE>B`i!;FMTxK}9i<-u^YZfY zbB=h`u0VrXpns)>wHBz$mVK5lfJW*lacAwQk+svfkbyikq%yilx zzSm3`|GqxtEL}^X()hN}EN@+Gp)DK0vlJYPJtK4(sU=2^W@PIjV@*ko96wz~=el{Q z8I!$Att2n5Ty`8o?GN6HC)qzVXJ)o~Dj?qVY5#k zSEh2@>#m`3<{6H=)MM^6Wj;cLcB)_!Isky2kYl+b4OO5&r-UIf?o0hsN1f&(erB&P8R_lGgUb2vxe$ zQg|Gjj7?$z3ju1&H{IrQg^|m~i-;?mw}LgPk8QYK8u#5#*E=LtuVhA(THZ2$b457l z#^bfleBQbe7mNJ1&as^}Jv$7o*)@uyVUW24DT1#>8~YUgmoY5TktXUBxRyP$_n@J8q@S z?gGgm*Uu=Tdn9Q;K{C{gk>lM^T-IS`8Ee0UX^2r3s_teE2j{43YDf@O(Y5 zBrSgfD*4WGY1R7Q#npEVE!&PvsF4a&Z0oIa{3$l21kt}n&XL#M)VK|DJ6JpzEzx1< zXxyu+o?7U2?t(}-vj^arpSUrk(`n~Aly^h*OSxMT@vKWAS`(R^16X2)Jmz&q^joHN zQ8!lX%T;Af6VtfaH&Q$P8PPa+gQnBl;SC{1E?I!eU#xxPCr0W6H)_`cR?YG-1zZtE zC8G!FQ+E(5@Oz-_-chdvoQBWJ?$fr+&ae%I{bIzD8l z-$|`$P-pe<`cYcv*>qC1IKA~-(*lc*8Ef0QXUMXP~Q#{$H+fUkjSOd2kViwdR zFo8;tlvHQeVqooD%^x$2MPbtj5KMT;+lVR`mG22bQNWSft54dedTC9NTC4zA0hLlz$8S%yGVpX@#=`%kReD;b8=8Kq9+6B$ z871p>THeWjr|^fsWP~c)ne&(XXT+xqQ=`pQIr}7N?;gWGFI~g~89NK6s*P3N>J=!o4l(bps7k_=Nd@+i zG6)p(G|oc!ajI~;K8I4Fa^^RfEw}K-aqQb3S(t~qkS$tW=KzE1oYsi?V&dN!@igDy z5i+P|^l)ISNesd=6JT%W;ebl8=v|(YIkvQS_IW`M9U%{wKRTg{;^pIpcPP>e z=2?ymjp>)ZNu8S^au%^YdF8BZQyqYYV<587=z0#rG(+_zeNRtkt6M#22M1{5-_j7< zfxn`OnaoO)Qu*o6^n6nXBX8h)XGruk6Tk`sv8m%>*J*3z5SD|VXBUs6NX8S&pXOCG{^z?Ass*=7L z2~1gOyN`3+)WQkhS~=X5ELG4&6>PIAi(zWVmRRD((-~0Nol71kg$_u6@gG% ziN9$~hpjK-{LXtHO}Bn0+v$xEdV$2Zi@^O8SsKdM4_g#f)AN*0y8V|0XA{J#98c47 z-@B*sCls5%)BpJPvHS^i6$_?JP-Gfkp!l<#OH0oGmT1}e_@eBdC;}ba@If&3rj0$_ zyp-QkF2^MChM^`LI8)yRlKF{P5$H9fb_IG`aF=(vss*%D4Bo*EQ!cu67(5+n-MwmE zOnIK>@j4xCSKoxJeiscTp^zZHmfU}3>FBxVtYxR*c;6oRbzpjadV!Dsk8Ja1R*fzz zm8LI8)YL2eddzX+60VjzmVUSUY4Q^1VUS+0T<-KmSH4M%yv%0(zUPsT#c0R}O-ab<+ zSXnu<9*3sMACM|Isir-V9e;-i-}Oxn9omqOxe#gkBauBOiCn637ZE?7UiG)(A4w6n z*KogSR)R)t58stE=hLmyD%PFQ*NQhf_K^imz@mpCRMOn363IOdxP^Vc5%O8~1G?Ry*+PN0v^)CnJ105yZ>qkd9N93q^Rq4d(HBpHbU@`A zokkRtCqtMDGi`=U?xuC2rW6ii>70aC`@yxflm}p$t$>XypuBxF7o@T!nEId8fqpro zR@3?oZicYYI3E^#O11C*A=%0h=6)KXkS1=P-{t4IQfeA;1o0x9{RaPnw_5xvRY$(e ziIX`qtB8elD&ab;qWLL05xAwwgkNr`500*z)$<)xXT#?qFH3}!a;tY8&rYZTY*vk! z_Dz_@e+ifj&SdEjMISmGk^e2nPFziNKR;sgtMT`xXdH{O_hNIN8^U!Z`qRe&>bQIm zil=}?bRolx4oFQvy4*k1^<>{%?XE|2TID+hcZ+X%ePgLQg_U(GvVqdc0+_QDfF) zv~g4trrOLtbd})$N7plV;LMZ+$C`w3G|h&mq1u0I3fsgCilZo1vu2;Ub{*`2L*`aQ zGJ$8n4E!>nzE&zlDR&$7TD?vjolt>9`AU*ob9q-x7Qnq)A z!F2Z$K__ILvM`W4g{+zuF83NU!BtU7W6r*UxRbXSK{bfcE3QJizpTJm-9e!>Ab=;d5|G} z&(CBQ=DdD^*sJ;ECnJdv|2jTaV9ka90ee9#6uhQAu*L_Gf$On;5^Dx56Nb78{@j~c zze6yuL30hwDH=k8wQ3L!j+3zMz^_-D$nsSY%!tieyNo3sF|~cmA_@UpGQ!>R4?(z} zj05GTsjqS7HU8B?EWCDmFUQizHjQ+L;ItS!!_G(6s)CHOXF>t)>MF5LP=B<D!mE}+$lSV! zj3d-yFqvPnAb_*8aAET$=xp^(y)evCm^%?A7#8v+M{`w)4%&*k`I;e#zmpxY+i*t+ z<~X;w|1xuuTOc|Ushlm0F)a-Sm$l9#ssq$({pwh;<2geTBp?U*J#-XQWqh)jFbfsi zHeI~9o$+(1Keqp01nLOczw2a5g1~gsEN>=xGYV5PgiBSPqW+oAXTLAiOC#2Vp1Q6= z<@lFvtaSMFmkx&{7Zb!$`hc1s9ePyinZFnjF>x}oPBq=JP|e4dA=581u!$)g^;FN# zx5N$fG3SqD&9n@OWI+Fv=X@=K0Zr|L*WxvteR(qVxu9PIqs#9qlpT3^2P`<{!-}O3 zzhHD$&7}|e|7fQR-~WZ4!)E`n&u_^2b@3nj{3e{cFQ_u#y7srbS#`dj6+F4o!>7Yf zKJb|itg0?v??E^f9=g}6+ZHtY!6Jrw0Z~!ne|fcS@{fIGCQd!*&=~GOy#N&~U@2lp z!>RhUM@_vOZS?o_Q4Mct>Ma-6lpQ*o%*%qjaj9MzCQP%Td%i!!Q z{Ax|#YfupyrkXUiUPT%)JG5&StDvnLWUA-)mN@3cJhV%istD`+np=qigsz>i^E)7I zt&dbYw5W?ym`^w*A1)2FuUX~j41lFoJ=DC8?YXW8d09(UI5W-Ue5Dk^ZifFKE$0v< zTCjH6vTfV8ZQH(O+vY9Xwr$rf+qP{RUGGKz9Ual*9-Z%GWDd{ZSG&z<3Z!13RJIj zS~}zEAcm>lB|Gi)LKQ~Xx^FRu03luh47@@xcqTk3gEVaombv<$tR!S9V{a{ zUu3y`S~^2rxz*0zwc-wN4ArA!!5F9X>3fQ((qQ)0j8!olRK~(mp zoh#&clP5^Yf@Le7_*$XBPbkd;0cj_ax=#WwBq}%Vx+x+auuhcbYnF+0X%5(X*$Di7 zY9MVOu8!b&eg94(a1Wu%us*nV;T(W@BA=VY7k&^LfB3VFkqx$gU?tgdh0qvyOqkfc zkbuXZ&tr%mu02JWcFbq*)&Z@%il5OHKSCrSc|TBAxtg{9RwNCLV;!j&aUbUHWPjc5 zZzbu>F0vSA4@ zH!Rk?eEr?H>HjXz7F9gtl`g^1yiI0aZAW5Zp$k;0tad$vnOv;Kkzz0^ubt&6nqB&Y zu8cn=HjkaDmY7toaBrxA6HH|Dby97T)IV_1k^A`^7@}=|w+H8qznp?nbAII}C~3c7 z=BsI8ZP5#nzs+)jtU%YIivvO~OoP zcsYBR3CC^C+uom#Yg~&BwhZa0X(70CrNR9F5KqctGOgwRB2UWjHf@ZU|8L0CAND)G z0cYBe=O14K79q+^r_Y92a%FR9rbpbFC+b(wmwvxAK)DFyOJ$Zu*Ga6 zT%SZ|9@ntG0+R>sbZp_QHlk^`m<^j38GB?SNkRWf~pK!ah#J;m5!i-aF+^F5X}{v`xzOu zs!rT#6Nt&08fNL`h_5cZJ4?#hF1nfe00hmc`Za>wx{uCT1bOvA0dcqL$ZJPc5nY#M zgUhtjBTxZdQ4lQ&IzbD^1dqmf2YGgnws4#ZC0@{NDGRSCN- z_6V4?#ZpWqMl>NCO`$2I>l9ob<pqY<@CJ4A+lY%{h}9{M49@qAsG=| zFO7OspO>k`$(OnXbtMfZxXlY`#D_2r z2F)KZVq3zRTr6u+f7qk)yWlyW`Se-NCg}Q#dct~RlP;pA!;uLmMiVej|4gk<=ORj-fVGEtioI4DJ~m44d=dZNX{X17~DUY)&_U zOY%p+L!8M3-$fbNT?zi{WMWlO0O{tZC1Q(`$=Nh4DfboE!-9Fl!`8aKj|26L?gJt} z5%e_#)ca>3K9cxc9)c`W&3;CKwjA-b08F04rN=a~KJXYOStJY1;_jvLFq3saSEoQV z149-qHQGzqL8jA$ckqAD{VFHH)f#5&Q>L!fK08oyzTf}=lBZFm?j8imAY9r(<`f#N z%1oF@QO?B3w{~-ejA*^+4QTGP-n0ZOE6YId)2a0-T@PgIT9Mst=T-sOhte9-{#rl1 zqBQuA_9Y+4zNehR#+tEvhWV9IsjgVkm7bQ!Cr`s~M7f@s;w}O&Kvar`!#9iJ+mV*<7BSSwVu0<@%D<0TmnFy$ma%uiuFJOyrvn?0HV}(z(5S0 zNsT0~e<5H-T245}Qk;R%#o+BznZZ22l%USMPcfV>RVbUJdr3CO>YdI2b(*~|Lz-=r zXH@MBeBeO=fpmv;NU=)IOro*F`(Nz-;6p{55xfgNIOJ;D9;DkBk}WotRj*S=ugCcm z*C0OO?b!H~%Tu2Nu8~~wNiN%fb%1-MvW#_MKk^t2$1@{XORX3$yXhF#^i@I1ol)Iw zt3X=pIKLZ@@PM^|nYrw!ZIgR%pt#T)!C=w{I%#yJW3)NT*~T|@XZZN??hr%+rVNFh<8 zA&Yp*Cp9cfJ z?qh^o-z4m(ap(u2!&Mxv0Cl_e{zA)Wv;U1WeSePA=KnoK{Re;kq|Nh}JP!OaPWar7 z|IWWuH2HVu4wi|`*$>mNBqz!D=l|Mj$MZjjUfO33w-YQe6I^y-UXwM3H2HG1VO3l7 zmR$#FrJdtH)Hph_ST&iQKrCP}N|<5Fpo@Z=#D-Xk03jUj_C72~&+?a&)-s^x(4Al! z-v|m2N{z!;OC%IRcPQeqjGzFVGp#HoLk#rJcfpL0FpUC7qaYuJ-&$aTv4jhc7GvnS z^4~9K4V@=EF=Y|kBZ`v_0D{G&EM@0;7@ZiK1?sk9xvvkB)!ikPfeA?n z{H5}v)HMvXfyrk=P1x6oz~}`Zie4&j8`j~fv$6%#X$h*~+S~S;C6$=i5siwe#3W`O zTFaVj&kKJLXr4nq3{yZUYi=nhN7=65Utu8SZx_W#jtqM~rh&#}?msaG+@=eGd$N=J z{^DlR-oQqLNfyJ(Krxp}(^%yN0Go3~ODM|lJ!6D+QMEs--PxQuSATbyR2xb5NhfJw zr8DWolzTYZAl73;AVJZGAdt-c6iK-UbYj4!sBjK9W12{OuHg;#wkyZDP3qjpN1I-C zcNCa&^WX5$efleoUYeWZe5W8cM(ddCI8<`zl|jOcwY4+m=9teP23@Q*V&esoU+RrC=a>#Mm6|P!2gG4R945@i**~0v$gwK+dFc_0&b8G9XSh(Rp ztkKvvxZX{)=Za7VB{aruP21@H&=Z zEyW~oWjy3j#kr3N3IgcXn`B0r+g9~N=b^QjWg>5G^;fZhXAg+Wg(hI1?R@InVMjRo z4Bh^R_lVpVnCaCn3LP^}OMJYp+t9|{NMo{h%-az^N6;s>w@TscQ6*Zvgv3Zn*oor- z(e=?r$U|KWozP^Uu#~p&AFzBod3CL``$x`dCO9Nc!kzH+ZFAHXxxGZay1#7iW<_r-I}t)277-Y99EDBIcXHa!_=lnB>Uvkn2sf z%eWej?w+rQP7#qROofp-^NQG3Tc2@*FrY9cf)YXhf;P&W^3#KMxw$>?bD0-BA2|_0 zf`ah5x>ti0NSzS5zq%NLyLQrUlTio`4d9f-SE=}#F&rlT=*jEndTX(Hw9X;S7#>y_ zB)YTZ@DS@85WB10WY4QD|H@Y?06vYA8PQarV&hT?<~20y5|VE%6)@B4np`V)rgEHH z?huIQ*v|U577u%DIgXp@9KjDsdpybr(VmtPTUTg5_8BiG3#(|q_8ANFgw1?7P7%4z zTV=T6zDHA>hX~|S$W@w6kvQ5CHuo#?)@YHbYl;B)mm;X(I7Me8;vlJWzyH&a#J+`h zHtYOz9iZ(|LkRZA2sAl=Jo+NjYM|u(Z9^AO?_|a3B<`53^lA;|wQhuBj8&ia>GMUJ zXQ=uW69f-^-e+3dytRGrI)6I`6udDfuH@f^MWMr+0(!GUMR}E+g9{~kwU&Y+I@2~= zr?Bs`ugdCkxAj_K%$CIl`sMnkoR~B9lV4?Wi&tz>T(C*%f0-v=xg( z7h-o4m9nv9l6OR>t3cm5b$0ajclIxn@72q0PDZ;XxL-$_&Nr{&4pe7)2AwdDnv>?zYT%IXMV{yR0-K*H<6X@zb2^Ke zz5yE6ICrGikujz-EJ1>*UPP>{$F}-tsbg+-ZuJl34Erdcn~OJjq!C66*X)myKUj-U zwJjZy8NSi|HaqF4nRprBS~(Yj8s-j&#FY<3DSWg%W4ZD|=XXQmxjA;urzQO+w^>$t?U zEt1KKm8DufH&mXam{yBYU_B61Y^J;S>dUXAGe588RJMu>B|v}zsTGS$IYt4F!7(EG zLuJGV4&)l#i3+rOxiRyAK=#N>~blSjD`#L)mTSFZU4Q73HK;yonP;Sg8wRQvuJp&={QfF=4p2_ zB^_ZsPrlZR{BpM}(Fa$))Gn(dg#Z~MyoQ_ew z8aspRa2+?f7tvlG&4ijv-Y{SS78tz>0lG;p;whTU^9qjp%JU}M$W6q8MeMZ2RnZ&? z%;E;1064%TV+hc9N{Nol$WyO&S-05`Gh@*oT&nR9_LxsiFJ)(@_ZnZ6nc%hjEpktL z?zS6|=1!3ZOV?A5xi$3yhoqTRE5lT75yk!& z>@HU6nz6kLcJhei;+un#A>TZ)etAFgqwyVvCPT#B1NZh&~HK=#TpY<`QLuhJ%>R8C_ zDJp{mN|Tv9pWAksR%|ng#ARf_e;c65`~Q7t*j3? zsigf_7}J+vIoE`Fu ztei#>{g}*17$fPN_<`DGn8q7!Y}WX$9B`i!yzss3i!U-XCnr|b6)|Hm3b_~|DA|3|FvYk)dPOFwO+ioxj=#s2~PF)}>)!q23za=>-9 ze`dp0m50xe)mgDbr8Pma0ty&4z`3h~U=_Q=oV85ULmOkM9@a)febuQG-kgUm%+Hz! z$T!~Bl?s#Y+8bpsa`Icm&9+R8-x3FIoQgI}c; zU_nf{?HbTf>|JM&%44;hw;6pR7|1*5t27vI%*03flS@z=4|fDCQOd8A)0+C6zBjtF z1Jp5!{i(#L1D_VD=t~;}fOU<}|8Z7&`=dqVqr(k^TV(rgBY0E^g(P z==k8#oc?7;zIp4?ouZ;$v>;6#tW0wjRD({vYh@Z9i}#kgTpM>m zPiNZ~^7y*j3o@44Y)^iXh8OH9ZY954kt;lv`k9KMWyb*SbDb4#$a!wD$k%t|!i`2s z4+Yz}k_aZj%DmL7lDu{3E{ml&jNsZx6XR?RiH>Vq_%T9IHhto{^La?T4xxn_-iJe)9 z9z0drwsl;ZmPhszPJrQYRf)FfCHxiV38ktPwRNjg9@YTLy!@Or6^rYc zs+!lSh_yJ2yp@+fRid6Y9l39NX}8L0FuZ+x-Ef&(rqYtacPKu+;|OO*Rm>BuO*2 zS&hp5+aX@AXOwa9AFS0MSbdY zrbgp-+o&Lm7d^U8VhOt+XVViECvMx|4sJqbQ%$0krO#BGncOO!2}il_2}eY_AwStM z+T(D48X*{6e|!wZyV6FfqCqqn95tgNkqm4DG3~xrdvIp3(AtTUSTT~4ocl4i|WgbsC4k*+v~Qc*GipdHRv;xZbD;Hd0L>YDjks$ z-p(gOHWTwc-Ox70e51ZZET@)PD`BFnyo`gt%(KLbC(Py+xnHxX& z!y~4t$#;x`#y1*vrL%)osP2&rdNLJJX@O^=79OvN}_v5vmW=Ejm)>j zm-`tURDtt19;R?ZEAQjWsi7-fMDC4zE=^q|*E*2=iK!-t6T@riYZOu7w6IR@3)S(7 zGjFiXk}zPF2FjxZ9jN=15*iQbPNwNx4{E{)OT$*9mnj+T`|b?($7KaX z6y68RFM%!Tt5JgGe6%KEh78qq`AHU??YRUhEX{0>R2UG(0y*nPEnVvIAzt<+=IBxZ+-EczruAvA;%s{H%Bp$exE5pB*~FVnYCc$U z<*%q|!Lqufrobya?KfA_tU@;G)NYVgxj|wY*na@#Z6=H&dKjqvL-Y9x|D0Fy3Z_nc zp6j*sWNs;a2LDM z!&@@dF5-RihoKYPA@zSRZK2nL3(-oQmQgy?v@`V5Z-hBZYQ8$#W?1KYpb_R$mIc_! zrhjazhO!h}%IMPcVOw|@N0FnI&&X-%+XyuU1}c%m|o1$*YfFSCbBAnu(NED zhO@kR)%HlrWyg2HW(CAu>bO^_oFU)N&(m4n-ph2<^?K0q%6C2LW;vb~=I3CH;no^E@&z|R(^5^u)i;c1xk=;&wXQ^uO-2{cu-oJ={4Abv+lB#g z_O0~zg#0p-%#ykvro-c?pK+kW1f7j$j2uI1C!Ltd2TPj0!+f<(lPP^8j~yOyKHNCAfdfo84w%6l5K+a4>K zMQyE78eQ<0)?GxYgo~))T;BA}YZV2kosJKflBQ{gjxXOQNKoA9>y~WD6g(mwaNhPc zVmHEPMXS2<-CJv}jft^~P`T>%iQQ`oQ1C~JY5XI@cjIId9k#;v!kT93&n1Koa_2wl zj5>fmY*4l14w~_|ltD)kycDH_Y^ai5y*Qo3Ag>laj`e+D@YMahs%pltEtGj64G;L; zji6EpL9b*-MNvE2O;sPZ_Y>8iQ!0h=Hn`k|c!h050AbfyD$6=*0m*vX9nsnR+bf`b z`>3Q`l$eH4OOyoWGL=ccEl&YdFO%`17tv0@)72zH!@i)(?d>aT1jU3Aj4x0FK}4Ko zMPmjiaKxpNePnujE}JrzhbNQDwG+=mNU&hfd^6mRoTpi%-t%|f%Gqg()e!4@+Q{q& z^1gZn%8F5lW|wnpTmhaTPNa@I))PNzJvDVu3v_ZTNfsEOpp+&~l%w3qwO{-r)>WO0 z%r{UR-wV0S9C88PZ)0L%?34;{T3`;<;i@#QU{Eq=aZT%rVtnkHAZ+qlSiUofgAWi%7dUcJT)L-m^RUq>nB^kMo zIm;Z0`E`fy%$>{$m71e(8n*pFBI^|TqyF(lxM%$Afdy6)}7NoIyd(=H<@3F3CbX<^sao%`viEXbk zw1=NGd?S~vM67!ZeR@whtkO-VNxQYAOWU)H&5o_IFn|j6ajtsyI;rODtf-rbddDlt znJb^;PQ_{(nD{G>7=GHbsd2xG6Ln}c9QM;ZV0sk1%hN~ieZKZtPkL1R$qs5=udP4s zG$Rng4^W?#th2?Pl{YbXkk4Fd6b5Ma=8wWEMC_{fG|>ZovZ>OoFEcD^N(cYz&pK87 zI{DL{J7BQ@-L9x#?#J6e0|MA%B*+2RF>LcJj~=lGYC4%DZ1D5f{Y130BL4K_Do`So zSj}Pe#g?N;O`o;WHxSq)wa49PK^N^n?MYip$FIE9q<8{hMb!y2rRa1kz{XpU3#Y~4 zBB$DAuf@Su%Dx)sY3-?-80?GtXP}it&dem&I(1b&=B&D>I$!^pG;N+P?CM!(x$c8- zR$22v+J>fMj9k-!QSXzrRFW5a=WFo{t1|sQKNCcLKa6Vf*@;uIhZVI1Km>t4^0E-P zukNAd)I}_L4-o~~4OUny%hIRZ7*ufEQ1{MXwr%6n^x=3iZJ!B@I%FYUtGtT?j$W53 zr%;abg{PRKsW~qD*}Ap>z`xU3PnE;TfvWT{)(T%01W^n_Vgm?i{B*fd57F=_ZKl|e zx@2@hLr8*7`>w1T@!;p=8k9go7OnZv zO0?0?Dj^keDeaimJ(53Sk5+xsLWBvn`X5vqt{0ah-mbqXr3Om!^+j-uBAkCBk=QP0 zF){X><=v#T*v``kg9A(p{M6fiol~Fg;?HfOUMz#!FJ^+St@0D{r_cd(RLfYpQ)8aT zr&a)0=QUM9w4`Su6@XV3Gh4jA^97lhZP!P(}rYI0Am71xZkZudb!0H8NwT=Q|Ju8+@z#R~PGeRm&s-h!h~-cVMg(qB*-s0*#>=YQ z?}f<{z?=6mY@HmxAGW5wvQdu@KTo(F$Bu+4Lgt+X8f~1DcnpHV5|P6%wYjZfPNtVc z(dWlLnl%BTrf@V8!uX{AN@tfB3sE`v(TJ>CaHqcL!Hc>jRtr!e=bOx#uC_*(jS)Zn zB~k-A8%WxG*r#o8csDhmJ--%jvtH=pRcHo+HVZn(Yq}QkNzwU}Nc)!3Wgx%gMlUxO zS=z4hlc?_W)^8U1PmfhnKmlFk#AYT9tXZ38ZU-K9q#kbjA^amY6@Mo%E(7YZRCu5D zj|t_t*in58cA6EDnkc234p_foZwHKeFK#IE-Y|euk~SK7CuqYWEfalwjq_}ufFDu& zaPLLCBt18)>Wxe`c~A*dBa62viBH+Q;Trd!Q!Y;FwRf44$0H|8VXvvuYp3J?k#^L4p*ZiQUvO%{7UWOEfJgkmZG8ch)8SMejfsNl<*WX=iFHqwmno@kQ3|o zgW))Dfk(h|vvJ=m3332S?^aq6bG}MPosWxAxUT|*TIko)ss<}ejjl$&|2{ldCbSn?h$#Pyd4o~#Rfe071X}?6pj|vtooLO3Lz@x?>V-kxG~ct_k~*%7l954ZQtw5+`py8(QiV zn2#u%D&wHBx__?h9;PahrBAkCR~t*6$GXU7z<8}v0x|{!U0!rjr$!4y@M@%WI%V2? zOI?S|o8uUIK#6aI#~-$frZg1K~vm@)q}#%?#~qC;5qbUv0YknO{o_!S|m9>z&{7s>5N{vrRA=*O9@jKFu(+d zixnh(P~#hvJ|EL7F=tWT)n8n z2^YMHk-#H2R9QS8*UNZ#KSe&CMP$={!};Sb&d0h%+f6o0u=r2vBCeUUt^TqK_(jFv zXj?nOj4k8Nu~{AOnR&Mmip%k|b-*Wa)@#nB612RqjwUUlW_VIOW^IO=DUn^oA@Izj zL0{DvVLhi!>)Vo59IxKb$G@LW^2qshSV29%pDj0K$8zpQ2eK=G@qUn>qXHrnCgV>V zhbJ(roVH1{!=2@`q+%buOV++Mw$geZOnrc{OdZd#nbdv$Jh>LXO9DH4zI1d_x~>U zY?w=jY#}+6OuOwGLPO(=iz|U77LifUM;rzF8*MQO@>kA`0XK0*$PLCAt5(B<1Jy5j zrzdeGTZ>MB->ip}@GFObrOK{jA2ZZwF0f2ar{2yy7<2Io-V;a>QEQ!JSNpm(HssMZ zVTa2@R|cRPw?IjVSr>L$h_vUgQn4V4dbw?0c`9K6#x9#0qLziWGN%GW_SG2k80{go zRYFWV^6s1iBL!RTH4O2$dLFr@i6Ru`rav}B-R=j`n4~&@1c))9;W%7^io+!K?Y%(n zuJ&F0W!go}ZPh_%YK|-$rg!J8Lu|8!5OESRQ!;i5$=l$!Y(Nf?WS~@zL!poZ9wrhM zxI7p?crQ=+zE%9laJM>^GYAn(Rm^jiTs1jdLH(!3dpF8uIM|N$jd`K6JXaDcU7~9~ ziBvBli6RIfD2%9`J7`Ch)sgPCdWxttND`~qcU?0!5lN;x(DcB zYAi)#kz+~*A6!SiHLT)8Ny|ynn!E9}i<54Q2!$;W8NJgh|H4w&N(Ls~`}T3PDz=hE z1p@F5SY$zbu7tfR8(AZgBn1l1%2JS!WSwukgakdQ^|lT^RD?@eGZWFhV5*Uuy^LXg zq9uW5m4hXS-|1W{tL7(IY@M;=!7dRI93*6ujRAE)V9V0pmc9DL5yaZk^Nf-dyLzfq zblH+)qC4E0}VzY^uvaDiS&@sx5UKb=W;yPEUL$PsSV{cK)`X9 z27zkm{5wb|E%m3l-Hq*>x!n)y>c&#z7M*o=r$L3(D~!|tO!hLYNJJYhvK^$c?=d-x zYip?~r2G3jm{-#vmJDEwy62Tu!o(vrLM$=pO%0`PM0I@Em&^i4;v41{qtU zBWZYAuB)S|Ue%8K!qODRqOzrJ38(v6D{f%{*Spl|x9cBxNp3BDXdUUz8De_o7R2SE ztr!Y!LlM8qIYH_35rJNf5Tgk%1|u<`%ofpglp4%`0wQl7 z6XKF`awQ9i$Op2sVcO~sV}1xJmy)V-Vg9cN-o z;v<`HZeEPrDO+ruub%m`usN;0>Om8h+5-mS_HB42Z!8|+{Bnr}d`s{$>ei&=vS}h1 z1m{O^DzJbPMrT@QYscwro{jkVC0eG|qEPIvj&;*b(d);u0q~eBk&CM>yRs>@V;L#i za@VFXhKH2;PM9m>OMf2fr0j^k=F?D{C3Jzyhbm~-j$~Cv!U&<+^s#)KXjQGR6^J`< z-gNd!V+AZKu3Liw#j}7PoH!_e1$qa)#tH*pY9AZ7Ao?S!p%mvGg_Wvf?V?bFQm*%J z;$F>7`0#X+LDUlN9d};8P^CS(BbsAAt=XknL;(>*l<(un;Cm`o02Zw=TUMKt67{-P zsq|ryC|(yt#^pZI*kQp=j4<^aZRS_z1n8fFYQreTz;|JRf79%40wheiQ9p^@>X!^1 zmdR!2Vr6pE*@-eFELL(=1FAq~OL_n9#43=L9|XT)hGS$3+#<;pvXulyb1nj`M%*!h?a$6h zN7Ea`>0*IhNR|B+z>C@Jc~3Xe&1a4~Pl2tpKOe#OF8H!klTwpMW|WGH&H^-NlOsAm zZ;7V%_8c@O#7B}&Ii+aQI8|yCOypUoD!`~H@CX%$>WiZ-rqb9vP*cLEEzN}hYDbQB zLMzQV_E9}?ja@87dDt$bQk%RSA$P(}^!Cesznx^BM!@@+JyML*_#9IHJ(aju0`z`5 zie%v7b5vJO?Co#*1A``JV8W#vMji8r1km|zU*<4W57p%SYCZo%s>KJy;mWGhI;#Dnv3HrKD%dcg%h0Ru6@*EM!gd%v4T}R@&hrJ@I zsLH?ts<>ZU=3$Fh<5Q5ks(M$Ssdu2jL0w)|5>-=+63Ce5(Ob^^nwVq+e(7zgan_7l z=3YlINa7%H)K=&ti_rV{!!;+l70S(rA*}-Q9-HCNZ zzg0S%0(Z53(|1Z^hcI$WKq3IFHtrt^9ZYJ+YQeRQ{>gL(7-9ls zbs3I|ihXB6jWOyluuwn)Y9&Pya`va$=vCQ@wE>00aMeGXvxG36`y`9ga<0h1g};E z^Qj2A>b$E`u)DacE;-h zJP(lGS8!?t2P!#*Us3MPVQtHse5o8KP|UCh9HJpa*v7(zi~@Ub z9@J87f8=!ntAQNRMb;ecQb(rF^Zv*v3>$>p2>l$&4JMRM@WC0u4Z{7?Z0QRuh>MN_ z1M2XXj8w|ph)k&U7~Zr=sUV#}S9>rpcbRm?Lfg;`>iE~tKy-*>u!pwk-9^r++?%3< zSH)9i;cggBo(Q?BtoN-uR&4Pc@j3aplI&h%>#Y<&QVbB?k>aVF@5BQ8{;=5PRkE#a zNXXO)hcfz}TJ(f-O*;Tr(lPA331Ni>##T%NRKFf~M#Wwkbpl@+to}ERGHaGF>Mf{m zIbjaQ;+7o;Zh~~|Hj7f8TzpExCRfo)QsvsFlnN22h8InCS zOks3Lf7;H-D{L?rIn866ky8NpHv;k3;gK_DI!-^H8RUAVb_=lINzXwmTma5sg-T&U zs_Id-jSZl;9(J+~cwr{KnyVkDsZ!*=WPMu=J|tJJY2hNct15v=Xd*CKh!Tn_=_LLCFJBoGCnamTq zHsM7_4I?1DE>95+MC^mdgR{LSq#JaONciX#8Y1DgTG|H!Vx-C|NJ+lPp@B{OPoZBi@{lBsU%i_zzU+bz zX@@%CS?D|@Z7i-afX?>j*ZucrXyr>sDR3}?fd^|WxMOZv?Q7;@>{9^n_yG?yyY`p2 zljDK)F*if$mjbpNpIg-@W|YBj|9DP^Caxtj33dMcL1VgY7Bs1T?@gXsRHuJMuG!Fs z2Tso^-(LlJM{s%U+T;#W3`-R%rWy#2L?)dSFt{}oWJWg)y~XJZXX7w+#f^QI1%^I&%;GYsgP$v!!#;GZQDHw9{Mb9B`Fr3g z7Xj0p!J_t}2PJtF9Q=Fdxof$5=XDUYq{P_vK>!%&fa7ERiVOSV+T-b4HBlv24T*>> zi6mMsw7qAsKTFah9*a{V5*H26efc7~k3TEBipc@;8};Ni))Yi;J|&PQwBI)#X8Wb%zu6s#7=g``wYk#A{=T&>9- zEmMW0kIts&K=U3`ilULR>3Nj;AH@oZdMyD z-zR1k2No}^Lw~(3>^gLrCk$Bc|DZYtg)VmHgvpH{5`=HTa&A6wKpFb! z8*^6}SAIC4sQ85%2X8Ncb3(bIo+ocEAaX+4%d8xhusfm*3VtOPqVty}I^t+)ZlwhA zCKnQLz0-AeP>c6Pota4e)tT$lO)oS^DBGU8!VvPY7kPBD9SO+J-e4?4V2w!=gO5pn z*X9!$AW?II@OuaaB*3*H!m%L$19)bMs>2q733Sg|M!X8MC1M0Pt~-|8Fp@9|cO7g^ zGZ&X>{2g9$@B*i0Q_ls^&jA^{K2%$6I=Q~e!2O?GYw#d!81BMpnPQ0-Ti1P=;+$t= zPJKC^4|+yRT!VwG?Y*b+**N}!ZHfTGf*rJeXng)K5CbA z%!ut;*C7`^nQJsQ<^|ZI>!?!qdV6fywpMP-ZFH6$L?42DM6~AXdB-=oS7H_Xqt>U< zk3QWp25Jf0}uyuq1w#RYKjsQD|S_}Bp$I7 zD2IJg!YK=KjVNgEpn~&n-|(AT({~8dy0{!%y=vl6V1Rfwj=F3&7sQow4SoJ&J#rfp zC0vt`kh<^14ClXjwU2Q<&mqFM-k58|{hT-f_Tp|WuG2aa;26Ax?fBCZd$p?EhBXU4 z?@8G}eb`Go=hnA_??^D&W}9%e>VzHYoMt`Z;3bRK7Un}TF)Ej|!Hi9y0mrK0>%nZy zVgR$r8JI5d!ackFQ#%6mh}r$YRmiMhRF5mH^KrKvI@kQeQ}e~WRcZ8i@ls>@c+vVh z#FE9$fi!T7{qr@`EY$b=UGVkU;i%Bpk32bDj1}vt9W1TwG|nauFMBxGPrAOwjt^LfvXwOk)+HGeh>Tk_uZo^^*3FII);&q>hX|K<*r9PN3gFe&GPfz zd3u3Hoiq}0&=~;#u7;}ds3U!pQ4!o=TYV!u5JbygWGebRc9hJ)R2`mdTUfLt(S!e_ zArS^!9A^nGexnbJ{3=dsnn^`g`ssI&xMl44VB~j4F%rn#&9lC)*exGg92+uULi_+G z_`tTF$GAWSIgm4uYSY@LL$KK!kIQ_ruAM)2**J#(h`4j`5)u|4D&4Z%fHfNkj(<~- zc=!gSO;&YYwJ*9{wb{50mUQgWGM1**A{ikHT{o!LYjfN1#y3G-wWv>Jsx_Oyh7Bz{ zc0_HHS`yCwDZA36m{nP%*{ItB{DNRysUhbSi(Y1>77!Zl&JthLfxLj00;-(jmp2AH zp=@3ns-0l#*fGy4lUZU!vlVzbNuv&29hJsp1H}l!6(D{oPg@x2{Y8;Xvs;RYmXU58J3ejO?{WaX`BX~*G;Lz+KB_cHQFm8&VqqaLK=r*Q5kB#4?N zm6leaS(1eE66515C+w&N9xZa%DQU-kNNT1_a2>b)jIIVS`W7!^n ziwYAji`B%nfL^YnWXy-2G5hZ0N<_<5G#=w57G>epZ}*oHus|>C3-vJH=(Iun0t<|<|+qP}n zHdc4v-us^O-i!CAvSwyRMO8&q#H^8HjL+e^Fnlc7^yvlqGiAkpL@HGrjY=zhfC(sFVYekBns3;ju+l{Z zdE9ZlAKDm{{Omh_%^Qu_Me*)#yX)1k-^s}?#MNS^TpkURG=!dtQX*aV*g55W%R+Y` zcRoaj3(PLNSf}EL)XnX-KThvdtN}Kd&8lJ5_;Ih9_ll2j8&e_TiA#RjIsIyg+=4^H zxEYc$OaiXA_r!NRF-&_>k51}Y+xBbekNBn^cd)^Q29K5dJuDT{cksG^;Y@kTgFdC~LGfR61-on4&7cf2 zOVKPPoBLOIhKq%Q+!?v^FSW%aqx#rAryrf#rZAF6IV2T)U*3~85xo{h{2;cd1!c?^ zPue#uJsf$UFup$eseN};Q{N;(wt}X;pu0(yG1Ox?bc{-n4+MjdU-ns(={l7xs5ZrUF)=PIJKTmeJxxpxwA9NkD6@*n%-aUHh5_{TfBaHIq;_P zBYMVcdID~iz_>BVLF^QdGcy&?Q%kvar(?yBeOI#$&}MTU0iGK~Eg`gGoCqoVc907M6d(JTtxjU;kypMWR*PQW;ZrZjcu zMO0E>-6HDUwdt73!XK-bpzk5yHs99UIkwV9&rO;Xr3hqF+D;LXc79L~sM+1kaN4>m zl^Yhtm@!UyKEAtZs37rU!HT1$YOSimr-EVgslU|1zeez`{3x`D)vh=1U0{o?@lI=0 zX$HEP4k?VNWfncg7F6xDnpLh_({ozix_d zGAp^=?BTRdBzO6Yo$R!hq2@MO#bCj7tq=V-0!2A6$e+e zu4ldtnmNb1Ak(JBR_Gb)JwPfY0iLB4ddWwBXZ!nZKbdR5-{bJ3VRq7dkN7Qf@Ikp{ zg~J~f&n6k%bPXf)xYo5NbFXt-#N+ayM*W7fGdZ-FV}c z3WtAFpi)&wF_xU7hUK@tb|h+LO4W_yU93Q`2;>6aO2=7pBzl+DN}~mlT8h7Si4l9n z^g0c4VVu|kgi=ni^>hRCI6ATxnY3PgQH8Xt7&M=xNRQeXnIore@h$}t zYrY3pzidOXF1kyp)dVxCk*{GS!~*WYE-PeYq$x?QIsFU6RJ@c5JR^DI__qGN z4(_PfYRVd|lFn}=@=C9^+pOa`V^-vXEdosi(rgFS*OIfA(T)-^!q7`4u|W5flA0MR z8OOe)^Gf5Ag#dUu&9lLvIHMH_Uhqd2)yx$3Y9wTtXQi`_@40pB^v*F}o=UC89c~Ih zZq5sXTY4!>SWiuQM0%S%xXL`)ypmdF8uhsH5Jx{7KeT$~6TIq0!irL<`O>TU&>b8c z$4YY5JWVoPMW^p+9>C=cqw?B_K2djK@ts<4c}Cu3*NK;7{>#fP_lw5Ds{nZ|aQB(7 zmi?Sov)AGW^ps*M+Iw)Ye6`}O&9Rr}{NY<2fPhnAa}w|YPOo436`hxk2T zhrQk3FPp2sPLAEtb6)Kgr={ln!>%6wzv}DZFR2msXn%!;QA#MtX5(VGSOAZFbCTGI zxdpLXJ5ptyR9QXwpF4aq^rBnAzB;u1k_d|QxC`apCUY{tSzj|3C8YvHuL_MAQU;lr z@l4Fhv&>!IJ02ftI!n!nWpds4e+knbGpMMa+mWg)#i*+6Ll<-*>aa(+@g`MSG#Y%W z5A#(gniEoQLQczi^1a!v!cGU>N0B1s$HZWu%e~+7weU0x~I2G!)m)D59CEZ zgf?k?oChDaq91^{A^6O6mA$rK``z`DD4MzpRm|8V06h9jn6%fu|6&XbB-%~+PvG0# zr}C#myf=qV1N$#+tY8&~5HGCz<@Rty`yU=RFLS@`!MiJ9a4##UrbU{6WVI?+>su29 zBM65zVSvw&2|hcmZHlXR^fLLhPMi$t2X~VMVr`m(V?6C-@awAJ!W4nbYAElLwgTel zWQPdC2T0GL4dWmE*y1u3zM|BuC$);)Rnr$?Y_3XW|NX7ORzF2!VHxcF?T(DC<>4!j z*>m1%-!NF&(j+V(aoR&&b){0@#}H%cC3~JAukQA$RvC1xHcr1~GfA*)+qFSamwHx* zR7hUWGfbo!my_VoqPi=v!fYPD&?H_yl%xY73-C$lctzPhGY+a-e-+4uQ$4!n+C1o3 z(8AJn&qnzufgFrMwCE+&VI;nQGR=IjS}oChRP_y@riXANF=0ZqVqMuZYHeguvIvoU zdu#ZjCwbwUj!ZWdWikS+2C39gaU=u%*AJbU1*g*Kc2&i*3Nt$YRjv|jM=~vJ0}h8P ziLO<4AWFv~DpBlw1JOMh$+RW5xZCMvo!?^TlKEQZ1$=4pDVEu#7YmkY@`}=kwg_eb z#VeW>V?iZZ$}d!b{cKy4sX0HQy`3AHKoFkN+?1VG`mySy`vz^}gMF{}qvss*1X`nc zE1psgSebAE9tswD9j605WG1txv`P@^Zo z7Q;LT)GKSS#DzYZUcMF?XF8x);)q(ou0>IyM6AUHX$nGDS$jXZ7xo ze8K#aD69h|PU_InCO;c90;6ZeE*tj!^$%$D6nb~$3oNmD%LuNOwsgr^GqVt$C3Q<& z6@wnn777&bZ7V~Z@JbCpRrM03canlI3@aCgp!vvD@Jo9F#1(Ro(ZM^BX(IpHt-+%? zLWjAcF#;MjBX#!F8czFL;P}LP`64z>moD7kN6CH51(9WuaV#mgCFWfgac*K2*K#Nq zt3!+&!RgYti&Jm5|IZ8g){MSX)+v1&8tKoY9r8MwW4*#$k*wnDP24vXQgzDekWLeS zcK(^y3SjSZt=&iIMPynyB%|P9b&=%C1wG6YN~LLAY@O4U_03-k$0r^QPkmDP+;)k` zaC=7SiY~$GYPq&U>bS1;DqDQ)=mzcpe57(s=k*{YW1g>q)nt@szK$W%iT(D;4sMrA zw|8!*(ni{U$qr#Hw^4*hLL_fKJQs*N-vejSG~cgmrTeppU60#J_tqoNa-q5LwL+RA zCOUubnFlZIFGca7^H&6li2k~(Y=%q_W?F3=oPO=3sYw%P4Cr_!fa{I2?+-gM#^u?rM~H6?af!tQC-!s-NcdIFIH68L%lMXB%jW7!_w+9MycaU2E z>M@EBN@2yEy1rhff8GjbtW!MwvMX}>ZWAmuBR%V{Zie+PDZnjM$ilow z-vU31OMd0lrj;2vV}6n3KSn06L|8>b9lw2f6IW9m&RIN8i;HiZlLT7G4~VZPXV7aZ zGKp#^oplJw$t4}QItrouF<@~)$#BnTx<5tEQAbvkyUSFIJGg0q34Z}+zxMG`{-WF1 zylE_=vY->(wAPK zy#tTe4(O)q%~sSS&}kh9mP1-e1-nWl+LoYrjT&u3Er;yDKvDqIxvJP#+;K9Fv7MlEs&kg;I^thh#MD&zSbL`bs6iAbu^5#n`7p1@vNiwgZc zxH{6(D9_8EEo`KCT}sjQW%8a7ehQQL<1wuY_SOJ`ii*=BI{S7>Z!VJ7T}Z9Ud8uhs zk%G%bV85}@SrPIWs-pr<_TMNr0-@+#AsMi($tg$Y+Qaz7dR6D7W9mEa*kV#pS>4 zyM$ejeZPpO?4U>ff=55C!8^rAza8(-Z8yJvFQ420a&Jfe<<2tpwEW9Gv3RMJ187Ly zp9N^}fFJs!7ha`@sdd+T!?z=V3&0fOEWwMK_7sCTPa@}!>0z z`-+OYUst_`YWlY2w14HB2t znO39PrEd_{-mCT+-bI+qGfF)t>^k(Bl(>naZ^%wU`E?(+4?2bUN~%^`T9cWjS-0fK zFCzW1vU9_hwz>fd0Z8y#(~?wa>hu>s;*}Its)Y&IQ4NSiCljjvK;V)nGF$Y5mvn8x zY*se68D@+SJ4%?9i8u?>&Bo%=FdFwXs8x0(rrKWIa2H3NHbTKMj6`WD6*?cm$Ri`o zy^m7wJX42cqQ>+gaH*%Tm@rnyW9s3`sA(4gpe=21D08(Vl!@R^s|^7>YYXHZ7QJe7@MhBqdv0kp@M#5ev|0?il32K= z8u*+}`Pc?u*%Y6(s&Yui|N4lG!K+>>bL00GZ3|1?wT-H}!P@L8Vo3$9pLIQB0kUx@ zw3YeIpK3ydd#BlgfGCJqj4~3BDdh*}fAH~3-D7)P&t<*_xOS{Caw@rg8`j%dax=c4 zDj${j>Vq?q<0harZDx1c0NfbY#O=%C+H3}1-sy?wj@~Ng{!VUR#0}PkIg-v!)xjA* z5%58unT>bO8$+Wea`WN$GWSgbe|MjO+vCWl7_xBf6T1`WFe*nJwfnjI8kbR4vK_YkYym3Zgnh+O_F;a)i9# z3#D}WYjXtY)yy;IJgJTPGsJ@js(32Kfrdk1$upS!^!>yH4Oh(Sqt z-~$l*c?e>e@%7E0Jmg@5A7`mJv;ip;Y^5sV?P&0iTs>ojKD`Nb8^|A0eVS>^1tif> zI=Qxjg;3{Q-dRTDnE{r|C~CEu4Ah0GRCu!rqrrdewtabwv0Y^JfY0tOMEe^uBKMY3 zGtsfTtOJP?qK_tDkgSU5R*x&*yYDhOoy7$f^_TJ~>yZ>-RThJxn(bV;YZN*DHGc}R zY#aG0Nn5lnb!b}MBFG{;JC_WMwcMuI?tGQgXqHA8sT|7PI%X_fg}aCW5^XWuW%}Gu zmhuJnw-NmwIEX%uGxjMM*Rk30AQRHw9PLm%+(ce?S&4D}?BLd<0QvkmnZFE{igOS= zooC?gKQG;Wb9y+1a=7!U%XeTN&d0`%RKAU|Q9FuNjoks^ZrQ?a$Ag{6+}}YS#p9nY zbAL@s88>Y!eKDJ>+HvDU@*~_F_XiBW*s~;7%f4v(ZC)S_oXrSBK_zAGBtWPs2wrCc zdM7>z1NIR%@ZL$Bk-vxGt$Dy?yotCNxE7^AeelvA4ik6d4168Y2|ro|3j&8b`x;^b zYlDbs60NvmAEmg$V3fovrupB8Lv<1>5g<{o34q3#S@YA4mcz6Gjn0++zR09BgM`Q1 zevzW>a?)w^sIK%bkFSkb&7K}TA1|aDKGcD*2ouCdo?x%bmX%O>3O4PR@NY=jV=QEs z(YNkHDSJ9O?Q9q1W7JBBYOWI59>ZvG$ZZ0Q)7xNkZ(<8=I^q2p7RM1=yRd5%LP(An zmmf=Bt9qrz;2H=CfaRq0byf1cRO^nkgr=403KyV3YMtgA`YxrY_TV4I$=V57edkn6 z>uE8QRAAM8UbMW2N-4i`(R$eu*Vu)4oy=mqY>ttIc+f^mM&i6|9AdmJr~C7!d=pqXj1ySg z|L-?E$StN>c)DM>J~g-}w8^~A+T=mo7cdYK!;en1OYiq7z}W`pFpBQln?h;Yp`i?-#0FMSWG1}HfiVMa`bg2GB)^(CaaSh46;LE*wz+m?z=W3vW0G?lvwUH=&3S?i@WK(m~ydP^S@mq0Z7^d$}|365v|T+M0Tl z;np|h$c8ydS*I>%c$f9UNPX=-bt{z(Z8>rT7|~DUyiNl8h^H&jCT9;v&Zxs2ZcM3Y zGMvGuRa9F?@u8JfKHVCxQMCsnDSdRw_d=ifvwkt+NP;@@%GNM| z(U$dgBH=MX*We&M@%|Oy9H0Cc1XlhPaTV^nMdOYWqY8sfqaZKhh&*7ny^nnJ(nC?+FBxK=%j&t z+Sg(H;8m-Pd~X89r$pfnh9!UK@3Gm-9g zc`;;rxh|C{Cgrm{jg)*#zT8W=AJNLG3{3Fxz*@~6rv221%Lw*fAbaRu(nK5g$jL@5UT=UW&fzzDH_ z*kTL9=hY6&+v)y!#C0! z``?*ajKkg0w^zpR?ccp#29*E5RbqXT?{1@DSFnDssd}5QqyWJ?xin@pxIj0#O~_@% zLvkvl9UISP;v;a;0k5bNUe$B>Dn9uth<6|2{cQ?@5s5cOB;TlDeB%9433nVLl7ASG z{9*&K@pr#ZXE2EOy-KAPsjN zyw22yRFo$1N)?j+;wb1gI*tmq+#aT5q^w$3e8aW(Bx5LE>gTuxFmf@|zR6c5;T3+a z*q;5~NHwvCW$o%6X4}nJD4tbKRVnTQ!3&ONa7%0+6GWmers1Z#gqz3+?lnb|FC$J znN>|`4j%=dE{h}~XZ2|-CCiNqkx8RP54qjd^|-lro^)A5l!@<0uEkL$q7&sl;`W@S zO?`cG2hyiOQvN$vcn~?_d;ou)IM+)6ijOd_fRUOBzYIV7LHQ+LvxW zYb)XpCc?G9?qwzAJ3${S7kYhYorx;y!}YqRG+v2LRxwe$S;QE!n#zMrm1g_N)%}%X z68vw_KgpJHzPe%7zcK5#TnqVq{=b!&36Ctxx`CGCCoZNN3|a-VlpcHSrJ8bXOFUn- ziKiUIig~bBu}&Sps1Ai=gfyshUQT}swo$th{Qr*7Rk4l${>_GrXT)-Zgk#hm!K4M( zAS9KRLJKD&;>L!d_mGe`RvJ^6^u*G|URgsRigPIq_6}!XoyEvEhTXw~y$-Mu3uHrT z3P&U{|A32BnlB$~{`#xjI0lYL3;gXWbaq8?bT# zQj_wZCm0uoPK4hinLRbHUmyn1=q|M4iyt!srQ2MxRYRg}+~O?}zc_S^_}Z=K4H)sZ z-md1XAx9=8^|pZrmP<@pv9h{!L7IVni6$w8EE1W?r%(9z>{{1A+nn;|$|z~R(hBuZ zM#D_9^RK~i$%-k{pruw7>>=>mblC(*()jl*70aYJBVS`ak~|KhBIg>qWIjHLd|{0! znxH_h?+#BX0z4dK3?mj5z9Gc3B6j=?r-N?!>&J-jDVF6*jNQKHvr2%!pg!_ z<}khC50&sQWusJM^(D;^gByQ#?#%~fq5!Whk7bTH+ZOY_{<2!ZJ}8TCHiv398`@|w zqE*N7Qcq-{z6ip<^7+B8K4Glfc3bZrHvcs1e)58-AW;`3)ZWg3Nshe(R$9 z@;ywiViJ^`-gHTDv}W8@7aSjNZg9F#Ue^d-ZU*7r&%mVGcNZsYbcIm)hOi!${rsn+ z43Ck^UY1;<$XV_Ab#@MCwA3%?LOT)(Oa1v z<0h`1`e8YjQ7?`XOfgw>5M1DnpJh_PW3!y8m2`2oGT>R5>jJXssW_>zD9qZPP-&0W zb2EN&%F2IEUtTz=GCXSPs7LDig?@Za-HsaJ9p1a=pSG*>6O)Q%Yf1%39#iBOir-9*iMrb(bV())Hzw zu*c_w$3mF`H2&WvR! zn*j?8Oc&8fD*$BBze+4uCzu56+#wYZu zN@qZG#`Bm!(*CfK;;;Uu4^?d6n0mF7TzE^T*o2o+KfBjDsE`!^tY6G8IqEC!2m~Og zYNGE2ht(~>N``zSp_CdETM|4B)2Xi{@3*tODH&ZRYL1@)=cA%DR=q)WcK_`lYA_tHmb&$u=j8Vo2*z>8P(_>>1|?_+JXJ zxsVfUD`pTJ=6XhPX$ws*ixmFs-wq~05`S-I^eHa!f~12HO=@DtMD?ffI1w^rtu#IL zq@n^>`KQ-je$uRN(%#lS+41C10~-=HZbO9JE!#9|eGutjQz@*#%$3L^(?bZWilNhj zj0EULqnc&m(|i^+$ci%WIJ2=}GRo?+p$;lDihX&cdaZe$Y=!8e#FM$%p801J@+ObS zQ)I4q=@gT%iRjjE2Zhu7RM%yv@OB6FX;5MF|3)`AC5b`70oAHO6vm4Mo$bilJKBGq zc3bgl?QVk9=K&e*1&wM7fEzvoLu(6wi%$PDWYTcZn?I>K_qpoKy;Yw1cRC6B9jeD4 zUxr(~=~>~R8M1v=S8lV>ovZ0MJM?B(qHlH4(UQSkKeqOq*s^`M7-3!h?xnu7lSf1&MNK^hzpd4?c zz2AxT4R_N^t7eF>$zhA`FGMFn(xo4A58NW`)=Lu}Zy@W_OPdwSa=4N&_+q}1QcAU! zyM!UVL8o_pg7z9JeC|}LI*_fOL$0Z(dHFLHVM~7U+NTvP{}+3$M1B%`4So!maD{O@ zE%|$bEjO+7!g1HIzcVk>a;M3sS)`+1chtBhggly~%7LM|zpbwK|Kb#CY0TAS2_kVZqPmR`Y8REsYF6(m~UKXVNR<1FZu%BgoKKjj{uoF{^p{3=bqTo0wO ziw`y+8a};Gw!Hq{2>*}?2I3phRQ0Q^%2qO^9edoYW+*k-`lzn6?E2maWuskco2 zSf1vyPrHuw9_~0e3=UaX5Mr6$kQmecFUyXLgs9rK8|U(u5K7Kl(r*sg$PX2M2HpB( z3if^N3VgHSglY}R#)v8rN%X5Gz*vJ!Kf=Ac>L#)_Xj~V}YsE3PP^yNfvZVa=V|!2( z95{5e2^LtqK#Wc_&2B@)rDTnNm^;SrT9DljXG#JSt2!gBUe|~~Z8yEt7ZqjHtc$2G zXOX0!C9vVkfE~g=J8zl8*YyUE#rxu)<8etZrff&IW2Z>+-BT(tqFoGX=NwQntzZC2 zIuQ#XTgPr2_wCL(M$1=Qorb4n;$JCGWj<7~u(iA|%hZa9u_3^RSrLzjcR?dwt)!K! z{&pnwQgOe9_hQsxkW05>4MIXJ0hZBC)zWYti&O?3HyG999w~@9!~sBcBUoBYC=C#z zkKY?MqhU9i*%nl!;%X?J+zGk)Zn$x(Hr7X+n#-EQ(4{sFh|n2eeZf+cNOIBG2?n1E zn&}~fQ`8@OK$=N3e<&xB!K#m}R&EGa6xt?Mqqkec@R6B-LDxeRp8{M6_`_O73A0-w zwdBzEcsd9ys($pq70yBDxjzgscl?TlmV;VbVsI;zpu0*)&h!al43)AoE=8|yl^!R0O zmcqaN)$Im0q2579-f4VFl%}qOsIz0ZY5J3E1~42#gg_DXB1fmqG#?C2|1M9kK#|NoxR> zIZkNoLe2uxc4Kw8TKf?w0OFg*h;qLYRXSNM&ITe|xO%TF#t=$q{BEo^9+URBh`x%% zX@a@h!EtiZ zCPF+r5g6J;5QcXfC}0eS5v^c)!te11%a2&{U&GXIsXiAjy-hax@EWIS^fxpD@;tC_ z=0nacL);eZKj;HgTe|-Su=9?o6CM&se}BKgSpDhmW$(#CDo4lT>*JNUg(DG7TKl7G zVz;zwk$`uq@iqp{?@wqB>(aMgo?bg$!Yz1VTgD4~~D#kqfs=;Bg0QbZNH4OBDYGiny7ZK_kJ~ekg ztOj5S?P;`eB!^b6EygUBDH@Wz=K|;ki7zAF7O|(-0m*L1`>xhc^JfbuVRM8oqv@g6-&EGeO>KzCb z9eub;eQPgS&Mt+4xj#6*Rkd(C2 zfJEIel-g=o^}IeZM;1az_P5>`REHP%IegU=oZ4YoHX~2PFs{3ZB20*M!+abkwIGIP z!i~AAa3>DViFXA?CsA#N2khsXv2IXHM8)+V3*ge@6myTDR-fR44}omRPB{ZTXwu}Jrnm5eH(z5Ges)t^^=#Wn~49we_tUrZI z_8Hp=0yX*JiD*{m+P&O1fxvG<_7QSJ{_c=L0>SRjeXQTxAxi1mTZ5yckFV4B47W?-fgA~Z&{0+5=U;8_air!vF)#{!9{s=8xME&XH}tlD2$kRH=cEN$ z*C@uw;$$-Yld%w$1`sXB%|}dzk^MI^Bv8D2P2Mz%dlF+u(s0lK4(SR>0q~kFHfvKI zEh8fwe2I^@uyRvdM0iw^u4<5SJ{q;r#Jsou=x5cpQ)qSt)U#qFG=*kLIEDMLbrjUe z`t4vqvl?g9M91VXQ_l7!X`oXS2CX;Ht(R*tYxhUW86AKu8*0SFHd6H?AR9q1r=T=q zYLiLeFC|Qn26ZvL3{mSglo+c(S@-^713zmI9?>&^xOYf!HH5fPDiDWZ`wz;4NX&woZ_t9pMDY&|C zewF1B%MZ($iIHE`9l9xG9e7!&FCWcx7cFT;d**RpO1xye;nEE=dbzNEgI&z%`Mmk-VYe0`FmC8)t$@&_kgQgA67i2(B8&#?cmT>t%|nY1VRVEw#jX*wgXIa zXyHsfrX?xhymAz*)CREam7TG2fSztz#XD_o31MejKtnfC2saI5?9c`|Ua@WdG#Niq zGS10dY8S@Cb9p5j-dYE{$`)tM{!0Tqv6p?Xz~F>ht-XLEYII1E;eR+R2|DoSY40RcDdW<_YF4ofQ*CjXmP?XLqcfX!A13ngu zm>^u_Z}0VMg@L(*Z{!P3T~<|k=5Zlax^-Dd7jy-*v$Rz;N#ahmb?)nh=k8Mw=>U1P z|2Fi-H>eNN0c@$?1$GO^-qUON`OZ)mcCdq_&{I-*5c*kiF7*>~AT8l2O=-o9UCeE>hsEM0sJw3L7iV_RD8M z@{lrRO}qH5-Zmd^jQ)bmXg`~dX29_bx@yb*J|3;)c6_|Y^CY62`7gg{Jr&j7D!AJ6 zOFerB)$Z}0B5^@7pLy3dJ1Rq63|~_I%dNa(m0eU)d>nzSPv*}nqnY#Ko2+VF?0jA4 z3SylZ(r&iy6x48vt0UB?MahWcIv1#`)imy&a21+=e$eu?Nh|L`8KAO|t+AV~f5XH| zLbP(vrvroapC@sU(o5uzCx0L}sNVhbA6A^>s7EVqx!gR{eb>|le2&rbARUW3rF1p- ztk5W@YuTv|qQQeXeCoQ2zC)ikeRH} zh=S?9$;22E+O>e}ft*6v>u+Ko)rCoH_g`&s=5g`Ty}3wT=GgrLccES+~24bSTgbRhLkSb2D6@Wq0f_w@Yi1iDP^-*VYV=JY^nW^zu0-7A068sP4rJOxU%!`_77A_ zK4_*WFW|>Qa@nuHsMmeEIo*?&el}^t9dU~}b%eh2CGUx}`B^(Y>icO|10zDfqQ93A zj#`_PRS?Wetu=PKyr()ET2cQCFg6WPl|^R`-n5%`l)tz90*q`6NLe34EW4ls@2W-$ zdw`_3sa+d7;a!IEaF8W`Vcb4Yw^bF`3X)t6Q(WK_VG*lHOdIPq*V5A|uAsasW^f1& zV+sl$@_E786sO9!Z?f&Kr8xqIwo<&OnOh9hnes8SJgOk{nU@{@&En=-(EKZAbzuMnO&0>4YZY7vz z&jwciW}HRVRc%nNFUW+owG4`5kc9gM8-kDQxbE7GXR7-*t0aJR%$7Y|Q`hzAWbJ-C zzV5p9X6gI3^+4(>`sFSJqdAW--;{21tJ#0D5~x^srlb_~x%_^x`wSZxwP3=*x4euy zIqT`|ucd+_50l_*1s~h3liyHrp673RW{Geh(#&E~adtWWI=J&60lSS~g7NeI=Hn;& zhiaY!=@^eA0X!2SgbdjAAu;9#^L61~CNH5kpDI4}Q#sp>_)#0em{f_4v{eEnf3qg_ zh5feK7+|XR;<)R8Ws*%q($Fe{(U~xHpu9S+r3h`LctgcVj`_TOkI`8n{sUerEo!Uu z#3P|WM@5THRX@;wLx}{$MA=!`U2QoOnRN(BqKYs`C!=|XZ5Kp#3%D^w#?}E_7EIeh z)u+{%C1kBGP))kd%G1B5%-evHk4u!ViqxW<8{q7GyHYoeS!Vd=0)-!Mui-kd(L8%0JjSS)_veMXh9d8$LuCeOT>HSn` zsa}>71?=nn1Ibo3`Ze@ewLKtjEE^`L4u+kSpDJr(fQZy#x!rg<$vY2Qh?%m9;W#nM z(Gi*tS^GN|DcoVC?QeQnit%Ltfa8tO&>b%eLo=il&o0=Nu-r!<38hp@UD9#HDGaUl z{Oc~RaZgxc4`RE};aFqOjB<`bkzhxr%ijTmm!-aNhwv|@A%72l*`7+t3W7s{f~^PXLO#^BHEo?QiCh&=(G z;m9WVCRO1G#QXA4i8hD=gfwq26I<43h$?F-4)}~Jo~nt*?G6Cq$LdMGiO=J}ySAEEI&; z2MP17)w~e{bdOpSng^{HE$))r3ee~DQbYRsScKb3F{-i?ZI3{IP>X2kLp-PSKoO3- zga7bvC=*HKlM0W%MCP4RZT-@U@+UA7gRfohhi+i+fsncug)+m4SW;SF)oR^@Y9FLFjrTas@g%N2d5Aa_UVh`=0NpHej7>RYXJFC|{vU*K9|tSiZ0{xLq)=aZX#`x^N(uVk!96LTHx)hxI zgt3m5tMYTkGv`iMd8t~0(YOn8!sUg3DgW|cZs==kf+!t>=|BuIJFIEFMFnt4%h|6G~#r+}U)8 zzq1&W=921FKV8b@Q`;-A?_#2xB#4$36K$tpa2`k8CnZGW?~n6sACh%BaTE+DTVkGO(q0Q73K|D6}P8)Wn|n< z?;S+lQ8o5X(k%rz`~v1mP?X~pGgNvkx%Sx@K^O#4QNTX$@4uVtDInX?dUzr^&<*Jwn3-4=tc_qgy>=& z2mJEg-sd+fpDH#<=Vyb%oUR zfCyqmWen&PS#lzNv1CZ%V^+x(i*R49B68Yv%qF z^RB=9dQhB#Hz5$I>;tl9r1(_-Re#wEsa|-&%$BYfghy!|#!LCiRzC8~nwOKn{Hp(U z)6B`;*#XNUG8di{b-V11vTg&Vhj^p4SYnvhy&lcc=)Ve@Y(oLF9ZXec56x5fZ8hk0 z4GPYhGHT>{!|DmZwJDSv_d$@d8(+bJF{!s=y7*{pxJ-%^!tvKmFkp)oZ2qPIp+UHIqLs1 zj5dRXgDHqNeTzB2vEBtahPZ3(o_)p)b7kB<QWM6jfGuuN_v7+n6kgv6( z-JAB4Y&u8!F=_7y5h0q3%Ai{-dhS>SCt6XCN=BCf&pL^c_Z;6H4t17$6?ztnUbZ z^TNKx8{b*KXRtCUihAR@VWqF-3 zddP*I{m+?D4YVV>m4AgZLE2l3@he#=(`stHpS@N6jO;KC+cBrc-PhOa`s-jE^#sL| zob+Tn&5pkT(|GB(Yx>IuzRsL!ZUpB$c3obt?D*_yu5FFo_rH`rb(j!y0jdAP)jdT? z(yig5E_d0sm1VQbwr#u1wr$(CZQHip#qRQ{|6gmJvG=}-m@zZsCP!qB$avoQeY{G; zO#scOY=y+8z4iFKG(IM6o(ZEz#6<;<|E)2e1Z6&f7|ABM4L z|E{wqOPir^)IibxjS-s<^>sCz)HjL-a0C*40| z8qSGCruVyxRz;9m)iCEhLWN8&qhaXR1XPJ?o_#w3c|ptdeP+EV%;>oJ8;D})3FzEa z_lw}|=aW1|NzV@ow`{d4ZKbc3^w6ZUk1_R~a4t)O>maMn-r7!wL8L2lKYQ?6-Xd3u zM@+VDW%=QVS)A)uLo%^Limdz^!pyMcikbbAC+%NtQ!+;OON(UgoMz>+1Kh!;?Pw$o z$lt%?N)9!CMnq>0Vj)fm-Vf0PQeQ7yo6$4A?~gl6P%KU$t4U@NEH^YY zGfJDpY0M(67EsD2{SM87!*o)^qsy(vvyI&RFI~)dhj{dvJ_B*5;F?Rv(|FG2+BCzj906gH0fF+faoO@Q{#p<5TmH z+SIW7pU;*V|30HCKPyc?eNZ!XqiaB3fEcc;poI_{fHNt~T3H(ERm^_V#Jjj0hvuH1jE}2ln4tP4c zEc_YU06Q@x1$(Cw?z5A-r;_CFbe8SmqN;j2+EZz%5?l5f z*p*+;aPP4~Rs)zz-$66gKFtRA?0g+r+3#ys&dpbL9CUV3N%7&a_xZQdZ9e#2mRfYj zd9O0XYPOloNqWj>hhJ6RBK}dP5X;Y!EtqkB)s_pGx^qLxlPbxSX;u~X*i%a#Z@Rq^ zlwVd>W~vlWT@lrL{x+l~M?AZEFm8X2oT`D!yg|;lGP>rwI+cxCunH;h(tqO+yF}7o zbijq05|+)j(qbLic7_s|%=+_B#(9A{(c~{+77-0Wxni~A>TYkt7ynGA*+f)OyT+3R zOG)%$k!?pi-d2al3)`NoHpNMR7N12c<>zK-m}vX!=Sg#(5!o^1K~7eP5TqIny8+OP zz}AN2G2qU}XI%Q-1(ghlm@<#5jt4y{Ik8lfmywIrErnOX(5JL1JSMJKUT|+OQ_IV> zyt1=;JOkM#c4ycc!wYu0+h6#L%&=7uJ+Kt~^E)45L2qR|2j6ZJ)}`uDRqxdyEg${e zbwttFN@=A!J$?V2hyFq`QIe#GTYts9q3C76JY!`dZ0iWk`}+^gco4eGJEVljbJaY)&Nes2n(`$DwB%RbblsfD5tHOmvLOsK;uxe zjBwcyjAs~!EvKkCAMUI&Q};D1jFFV~opw+~NVjwP6AfjPb0F#A4+8t4J$o4iw-NG@ zqG9(GYZIxh1do{~Gag;GLCq(V7EDKGe+yf(*^z#9aq_w7yX(`{(~h7k@?)qEJi0vs ze(VMYyC(uZF6W<}ZNpwy-u!n8V$)vW@H_r+GXwq+ttXOFOH#A%Sv~jUKG_1G0xP77 z!H085uJtW2*(?Pg_~nV``3hXR3L`!~%s3zb(SBi*TyD2x6H-vyh)#sjtF5QIy`|UN z+4JgDcgwEMoi9}uDlc?PtSPCHBrQ>pkOtxUaW*>GZoIvQNH%GCsLCf!+MyeE2i7j+ z(ajK=U?Su9QL}SOIK%NwG5?9>N>*+g6v&>+gkQazJg^9E+BpYQRwI+)KroUezU?*d z%d0g|Ivh<-FZ%(1xA+sXo>fC0;fp`Aw_%zBT^wp#mnJQeEdR*gW^T;epf8x33bq{U zp|6WksM}Ggm46!gZ*+#KxuCImcUm{~x)&LG?G2{oR|LpKOT6=F9xuDwbj!P&_SCh)liQ=$el({7u;b}M~j=T zY;_}&XP(0aAF)8K97dEtFa5zU2o5oy%61d*gOT_&!z1J+f4xsq{S$+5B+Q{7^gp)x zZwP|1l0%^Pp8&*#u@UJ18-U*JrXh1%3fI`)}~Uf<5I$LcsdZuNMCUx|fdn&6P7F zLDRi+L6B^2>#^B=8wK&w>VlL-@V`ohRPw)}o`R0j=F=>-+t3F!ag@yXrPub$tMkhy zrDFAcq78iX*bli4e`|9Wq-)TS;ON8Rj-(l;E25%MNwS^F&9_;E&O4lyvz^MH=m1lz z$IN@MXh5WAkuO(2^*9sbuPTH&9L>SFtx}A~wBG8SKA(er8mis$uUD{#=9!&Sq^jy8 zkq5#K;{obW2WRjA7cRM0V_+_3XtBoQ7?uz9BzO*@0e5$R);h58ptLM%$uSQ~_l`s( zXyOE@h+w`H;>j%?KBmvUF*3mSdfD@^sH82GCdD$ZQR-AQUcV9ID9V81~JjJ21NI;-L2tTUJCvM41E?$V1H`95lpikU@9!noj3f^(#2 z6Ip=(c-%a)ZJIx-pz0hKT}!}CwL7q-#KQlPba3*P$5t_w5MU@zT^)-`VUc52-Qb1f z#*U_%KPj#-pImvOf|5a2DCrOiP`sA8@^WPN^Z8K@ac;#89b8|e!iCZS&F+mJtNyww zc;7N-7l?&IDeYsr3=~2DPfKxQHzxLMGl8)emYw1q%5oB^M;UH2*V=f}M(&N|LIkj) zDpAmjDlXrgpzqPp1GBEo)-xXLfy|3Z@tG;{Mj{ll`M`~SBmVA&^v0R-XV}`mw0qYN z!iTiw10BoKG5RtI)y5XqrGR91DLMwW0B*kry7M9GOd7*IJhzw|FZ^0wEF@ptsg$ch zt+gXeMIf}gn`dxH_m}Hd zOEbNP&yAk{aZa=G9}7}$nZ=x}o}J<`*$S1-`S_!w?LRkKP$5H8dtpL#K$I4*ub+Q9 z>)81Nm`Q5hO9^-4 zK?`@{?TccKhT>SVFqx16AyI>oL~nWfWL5vDKe)Y-qKo1W4O6)*%zF-Fo`kR%u^eck z*eKC)5~F7u?8dOXqc|klZ4L+YbqrpBAX?uFc5EKnb#wnJc5s16j^+h7VF5Z;c!h`$ zZ5A`I&86BDCm)@igt@YWOSMR{wZYH{=(7k+Pr9|fHCza3EmjJ8;F?(}0DAWkBdVSd z|12C-nx|Pm9G!itKm|9Gv(fq3GFs#ijJ5@W#f)nZ5ycFqIR#ZL>XGR=y~r|ncySGM z6`favsRC?egrAyh? zrhW+87G!W5OFd2d^9`!qGU56lMN{EYx=DH<#%r?<)&NlVSTEQx?Q&oFE-J21JLRw2 z4ac!EfJ^NExPz@t(C1LFUUBJ~9<`iU&-Exhaxt#Xi&g2F`|pmz$oYNyf1}w4&J1mG zBj>+Q|IUDR%Z^;^JoiG(aAH*o(fgW?T!@33J$17%*-DZ=ldY|}nNAPtyc=w;OK5r9 z>F?S6%!{;W7TsGp+rmMCeeUxA`KJa(C%miq$eo-g>Fo`{J1acc#?^EEx+A5%@5DW5 znl74~V*TQ|Zn1d1Lea))6TMBmb^Mug_Z9t|d76#g*gZ1b9Mq+~Y?*e!u2+W6+&(h; zTiP+2T85!A%_F6lC`t7~sXDYkPFrl7Gn0)i$s+nBE2s`w54lb%VwznPC~dp!1mIP44Eh6k0(O_h9Q{Ro#FfPu}*1$urR` z3(KU*8T3dSCE5eKG>tRmW6mU|0TQK?!%HdGg*kV4pe}Kgbe|bz$nE3j$;VM~9(7N~ z7)!>wFS@R*0&t66Ni*>gj9k1DG9;m#B$%1WGcgyZBJCFsh^8aLi=qL;P|$anCT>B* zJR4r-;iwub%KsR9^kYZv{Tt*k`>^(3(vkst(*a?g$F=>KvRf!Iw74wuy9_iZ$7v>o z6JXhYjD6h^TCXWVi+#dW#HKkbZ+8=dl5=`nmrfoBNvuuo`5ySu%0V6p2{dpd{-yRH zCoI^M%zZ{m>>NX@ty#QwE~RmEvFp!8@#Tq9_ypk+z!|c7(B%oUdC;Z4EVAap|3bWC z=dOY_{2qE_;mX9XcSwIkC*X(YJETd#H+!nSmD=isT zr9}LzLNBBud3Xc+Sea<=QO<6*$Nk|ss4IV6zWi6MqJ21_r|(q4H<9L7az0hRA|3PgQBAh9lP%eKV=K zrnYOmJRPnd%uj@&)ah)UweuXph4E!ZOtt;WdJ;V)2^ctEC{36L?6<+4`Rs0Z zt%JTFADLG7pKmyQ!c6P{_4$2QLTSDe=ASJstb(-z0JoWTkUUe>j9888t>gfY=XW9)@EQ`;;0~x_!0s7sFKty;)pbFZJ zLmtW3ehHVy&F>pJCC~DZwMDG$ZBCQOJ#%?c>W&y2hI4a_8Kg1eQuk)}{GYIDmumSl z3moPd>XC|@!zN5BJ}hE=T}Gxg;OUNFC3XU#w9WWB`*U7Lc`y(wP!9Y3v!Qra9!Cl3 zHPNJR`0Va$Jlo~!D?S!7&Mg-?gpR?0ZQFx7hAfyl#5ugSgMJazdZ%i!BTwk~b>E)i8Jb=S%OHai!fuYgeJeXhBr4ze)w^Y?WKojLidlQG44 zTEDcn;f3D1fc4^TL+r=dZOngYJtB;CRaBO7AP0jM_1^N=c76WFHpox0ih5|v!;{~P ztGF4OVcPO9E%7P(%VzLi7?`JK-GRh#QC;-yJ6Vy6jiJC$C7oC`*8p@CdDem?XTyn( zpt6vlO44BUFgBM92xw&kgjc7$$)@5tjz;LI!I( zVFTWV;S#(==cm+z(uD0V2?f)CauP0vAp^ zENVwUpHf{SAFys>R)GnHJP5GvXG8u2>xv7C+y)e?xj5rAmkW>;u}^LDUefXd8aa(! z%#WaNelt3&nQVqh+CIMCcfQ5G$!oj`NBA6Z->}{&BPxGCA^JzU3D+E@{OGn#@?gKG zV$#;wXA@K-GAlby$~eNW1?T8v+LIOs=%LoVeJKllI0RGTDh3@^XG{^+5>P~29&BI! zb5(s#5FFkOnt-M1GaHE<<$S4$f)fzj;!ysA47Ma0kCY@c5BHr+!;xKLkkYS@fpa)M z@mTjlVY%baZ8o&LY^k@dOIDqBB`f3{3Qp#fameBrQ84YI|ZoO6tDs*pJ527MJ+= zwhKi@Z^4~;xcp#b)ih`cQL=Wh+;IA@T*QA}y?qYQuuRy%>FW7ESMS=o;b`JKr&TIS zpfTbEH1=V8v=r3#cqP5ytG;$qPP55sUq&*WVu|1z&sS~I!m7#uj1@sPGy`Rk_Hw0rDxgs+cS==F#izEyb(pPKAm$N z!0|Tc{^1d2_lUH~nNH>{FCDWdP5?g!N9D zm}~C(87R7M$h_sA_KG*tWQptvruTV4`7~g=UVmfS#iiQ94veA|WDc0>EDO!eeUDDU zsfGOdyO)@BgrPLULNaAvuSn*28vM`1cK*i4c0cXibeJ>l+)?d|J((tE3`cwCQxNvC zQ-)Nq=Fgne_BOS{^qbhf1&6Td2D}-mQpe1?*eofp9+m{B&Lc16xgM!v;2y3ZYSmS~ zrm>MMO^1nYhy9#;4%KJvEZ!Efk(ZXJSL?2~HM1S`nbDpIRDVo=p12A=u&Pm8CqTS5 zaQ4mfy8etHTdnl)NeFK~tx2QmQ7&~!ot`?=6Mj85>6e;41A@9zEeixTord*$OtZi~ zN3HJS1?#&~oQc@NCT76h%sMU5oX66}jGwWY!905cC_rBdjiwn@O6Kc;Kp4oyU`Z7j z?cUoXuC#P|?udb2l5j&pnlqntnP-hnd0-$1-|7u=v0%@N-v^!;=`nAOF#ZGUOy&Q9 zb&G}Nu!2!nHN6N6k$AdqSQi`b_{hfR-y!p6x?Sywq9f1j&r|iJJR;oK*?J8 zJn6n?U&#M+UWoUAxt5W&?0wx#;kAnDOyvNJWf$P^;|{^*jO#13^G$2K02PyYe>9%k zPXkN_?`%=iiT1Ch8LIiQru{KRC;@vQJV&&*d8pC-Zu3GA+s!Hbi7<*I>?RdXQ%RdP z4egZvjoXa)3YzOsl^nGgFopRM z{gWGB|G%tT$*4*b2x)?boJU5T$pcLoydGi?uDhJWO0W|%aWb^r-$CF=wBc|9-M8SlL(KFED z(xFx9rm=wnF}Yww_Dl7+Ccb4o>$Q^J_H-NY!D8lGy3;WQ&4`)W028w4`yx6tnF9wk zjDg%zB;`X4u#e?TN^x@P z?&MX+Z5(nf$7RP_nK%1KZQnU$EjX+5lCW7{=$C@mAh|$b+$a zI>*XeUAVV!4cCIQ)hkxex#S=VmyOP_=z)X6X_)x!gso4=K)f_^w`yzg*isla1%5=o zQ(|}g$hPCy_Cq_Z#R`zTdT>(6?L#mSF7!wJc&9Uixz@VHeVBH$pVt6eVCui@Bau>x>kvXOVyovb^dDPKj(hX@<<>>8(e5-`@gn7@UZ2bx5oqO^ zNAhj!&Vu{a9${M7-ru%vj}*Y&hGtSF?Ik!@VpC?@NhYR8!26nYKw4aO#-zw)X;rgH zpOBUB*N+Nm@teCXIsm2G8UcQN{N-;|`bzw41R>@((>)rZ2o{%3Ze9Fi>)R`SyOGF2 zxKjeqLDdI~;YjSbC(GlNsRXGqJJ!GlSP@-zYos~RVzFarYvTCJ+?7quPt0wV)Ih#W z6PQxe!D3^Mh!#;aXC`D_m?{wDwAI^Zt`B0_HatUZ7H>}bzik`r{@^~qM{F_Q<7(kk zg^76IogFtX`aH;kKTvWj`98=i%;i^N!VwFK{Yk+p>1VeVxy2kBzYmz`mj>CaC0ykI z0Ck!i2l!4A=2K+a!58^izsJ`FSpYqY&v9umgAi7zL|FQa6=lTZh8oOjm!;$tR znSWST+C6eivb;2Mr=(GRV5vk0(VSO8XWV1u68c^>SXMV^#aSonzl^1cA#CFVVCqPG z?FB8i*^Fl;HCWeFv?~0$Iw2ah{uxFoK_tvty8hibnX+C>3j)aXJmFrk`9&OqJDzV) zO^7IEw_uZ!1FpVlhb#>6dnA^+qqRZ4py|&8@1#oD@n+9G6gR=>R#Tu)Cn{&BDR@@o z^=;|$C*0M*44AHvik7G7PUO}#NW76-{Ddk^EzB>KM}%H8y9jP_ve|V#5vn6CF0=A zX)>u0?_zm~>y=znyww~(zV0Kv7+1Q~vnxmwR@tne)b(k#)H6a`X(AO5Bf_&{*JpO@ zyEW?{(Bp2ulz`i>3Zkx@Q<364x=WUjnYl*EbDnA{wp$?A^ba-y+nUu>_pVCY^799e zvBU*=cCz5eGPRL9PF-n7X3=wX+HjLM@~sWjHLf?DaB@GP*UaXkpsqyd%Dx3%e#EFE zu7Olf-yisdvZ~^(*=HJTzDW?JHXVKOdzdLV7RQ}@{ZvaYyQ1#X`8z-Y*v=c+-YjfT zqu_`kt%S~Twlv3x8;AMO40}-Tl3v&#`Vgbr+>=vrMC&fKDxuDRivZ(hyq(*CA`*ww z((divo6GkdmH)kieMwzgehYV$?(Kvm>UQsyOxBW>CK&C?SLCmNg0jBsJ$|eAC7hkZHT$S)By|I z%;fLV?*$PhB(Wt_tQlk&oKREKT7-9G?VT_MN)~GU-496ucP53QH%f5a1-fM@&PJlZ?%l^tE$+D8=JVvr>W%XzQdpzkZ^N0xGtn;6H)?`J7-MKda3YRBJHMM$vcsp%aQ^xOtGhkB*J?(=ovLrw+8=pmPb{5q)s$)OIcX1&2$~kxS6m@nk=n2{@;yTQ+iQ?1oca^6q(|m<@l|J3(3q($dXEOPxN)X8#CdcSyEPKXo@8v5zm8UD zU}xDX3s_lYqW>htn%*y!Wt2OiTIB|9zWb2>J2%CsC9uLy4@JvfRKrxyhLT*Rue`rI zuFV8Z#Pi3gv~AZfs~wafY;nsV>{FO)uKlC4zq7?Y~r-TK_{9;!>y9r7kB2H4e zkcOpdC9R$ku8zXQ*)Ssrrf)Nl_A)qtLDBnr|(U|7vezE2pZhQ6OEdLm8$a7GKH2#(laQv1e%sJUk?HQgGKW1ISs39dIPs_5s zifzbpsm)9@?RA9U@$N|2VKsw+2|N&uCv7vg`)%2bYG851;&mrfSLLMx0EceH4Rn3i z3Oet^iklMW*#h0@tgs-QA_#Uarcze@*pXlWPHPgz89qa0Lb9iud~K~344b#u-k$K1 zmH#PfPjgk{u-3_)@6?RcNb$`mpa6Og6J9w&Rz!{g#Yb#Z|0K85^*w)}{4$2ofnCHb z?k7wF@vrNtK`5Q*G_=ZFt_m7QO{&XT4P#=@Fc~PF5pV!g%z_07x&%_gMs>7<>VQ)g zX$rMf{5R8(V*G)h)FJIQWUqiVB2`I}9E{p~;Jk=ktsV2>LQhW{?d5q`o1a5Id7vNQ zM8-C8Qnw-&BK!0P9@F&UIkl3UPOs0RG`LKgCzWz)5G!gNhA>c-zo({SkU7{{VGj31 z8I5t~3inKGWB(pM{LYcsdQ(5ZPxO%Uw-e*ph<_hIj;S_x5RZ=q-3T*YZS8vXm*^3X z-tmnloTiH%oC>F}E1>z66jsY<@?p1rVjnc3zBU^$(`xOA*PclHM`5??XNDUur#D}% zmW9KssM-B^cF$0X{zYVuZSHqMX^rxddv8eQe1tsVkYS6G>7Hx+!6Wd_@kX4(3_a(b z$&OezG@i*=d(ngh3 zWY8Y?cS#gjZ-j`Xgqn0yhg)ha$BM>fR=~q*)CoAfl3l)C%dR6D%gVMB`%(|c5BP>3 zw^X5a6ioT)OB+tShN#y0$f-lJF9GtYjJlJ5rG`#cR`{)ohPNsnTJ6d_Cfb{hPMjIiFO6lkobz?v+ELoky}zI1&|seDPeyL7&EA&yM^<@bBc?0-lf?EHQJ9IpL)ebvWM*xMQ6 ze~&xnQQLot)clb(jnK{w&@UlO!{7Lj4Ad_fd+H`xg?cTn!CYb#Ga(0m8-|@g%e&y% zBz<0rp25#EK*TSYMCAf?+GltvDYD8z8lP(2982p5cXVTYT}xJks6$_aiiu)l2~3qL z*)t`>*Z|K*aCv{ot;c?mb9v>{;Mweb=TU;KGijYzb{%8x8uVz(!C1Iy(|s#d@*ABm z&Aw^<+_ifPKtf38>g=4I+MPn`sLx&eiohlc^)frE@0tTetIbikpT9^9_pm8=3 zk`s+&4!Nun*G6)=3At`xui+07d2DWJ0V-6di=*lqKKOw@Q9zgmMgbz9%c=9_Ax5N- z3ta9cYStPU%CD?Xcw1a6nV=_+d1TX0M;qq+h;@cKJ_{NXrxd-OcW2H`Zw9>Tr1h>L z)7&m$+fa&~+%~)YR1U}cLXH$uSMrEZd9tw1>q2#-QK}s&Q8dM6fFZ2{*CY&4_A}() zyL!ACUf(TeK$x0IL^s*PL{KuAy=O7=_ONw|!Ky5HVEWyOV_i6;+u3Wsc%ZYRo80^V zgvPsjl)5xJhb1T0&|X9#H9&dpRba3gnb|_;nRD<@SivN(5Ght;(;8zs^)Oe4JU2~k z=UcMbwHD^+U|wxP3Arp1Bx;RNgYAqLm*F~in41((%8wJRGDZfhrO4nhJK z!%C6DyYlgjtrGgRMsJ4k4KaWmVKA#ksyR~K+cSvF2@wps5wpCtau~lKW;q*~f6Prf zt613`bnCT1N)%R;>J014NOc%9zm01=jB6N`apOU;%Vn6nh9z}7a2rLou)({~*x zE{&Af)r6^MUiOiXwTwuMr?~K2e>5Tu)%=7ELACwytRF+YIy^GF3z50Vt2PyIWnWk* z*wB)GMPpnD{PTYOIt&OpK0er^X)HTUFJW{4MO&iU9JqfYDb;4*ezehb#C&L+zDFA` zpAeJ|4KXw1=rQ*zH?{HWukTmvsnwI*)9e$U+Sk;V``7GrWUechPtS=+KiZ^lIu3fw zm(D}OigbK(Suru^Hl@StS~W3_XfgKSENy(v3*Uh2ZYQ2>f}^t(!!Alq_e;ps_hXrx zy7B$mNXigBqOsR$f6Wz40RW0(JWGep5%X&^a zN}SHlr@>{dA*c~$I$>t#7QzRA9UoGCq9KqW7LF;}2-0aMnMm2&6a1Bvij@Y_QDfcP z8?sa_7z@n%ncmC`CbJMu4hHV+{7I-6)ij-z+&AMj$!g5@Qx6dzHZ!(G@R^6xN3u$o z=bhsh@_s}1YEOl_&E&2Mm=Ye2tL!^H8gZT1sw$efnTja&6>5uMmJpN zvD;`L%mw7jbRVs^1?O;M?E8U4utf8$aYsTG9|CyH5Hu7rRq4ITPv7zvfBm^Xilmp* zeQp*3EM(BbjrwJ1gKI5VgX-A`DY=GM53e+ds{&|Oa{PPNzo+Idat_dQZlBFdS;IXN%h><#4F>Ylz|W$^by)1| z_nRbR!|oE3pMe>|%5;}IQoK%;{y`YY^ci-75Bv0g9#J|KF$8@mEJo^w96C85 z)q`KwUgXpdU@WgX#ze z4!0RDt2}d@Q71MTVicR++p=?YXu+eFXys3*Od$u@)l5eP%jdd5_15yMRx*IXQG_yaF4`6RBk6m#d^K5a;Ap8FKU

|T*3MHy$3wxEAOYV z3LzqFL_xOrS29e!MDm#2ShG5_14YC}ou7=!m=fv%KKV7>ncCp{cw1-MP(`v zg=39Q)YL=5wopP0F_t#2sn8!7_e#?heU7^y7JLr@@A$n+jO{09@7az#^xfV*zRpR9 zcoes0_daZy(^EY2Z^Kp3d*;x3;D=#jHX8zbym-y~lD44%F_R8PX7=}0r`PIHzc|>` zSB^CfK3SBRV9J8sCr`-D+xxG;aNf6aswE?$%7GvHe+(y`gVPn0pDZr?{=UX>1Yt}* z;^jvT-hhxD>d&q>T5@5!Hw}Dw*z=>d5hojCO=_gKTPLhgFZ!zC z>o4y_-b;qHiF+uV#POc_x3^Y;aW!ekTG}!cN=OmCI!Ax|CN^8!i0gB^jO>uo)T*d_ z!3M3v*9&S;vcC9G8Rl$;dFbQ*0js-2P9|fw-?ElGynm*=sN?Us)`vrw)oTey(dd!h zykNu8B2?k!^$Dq}tLsJkSn?rm_sERhkhBLMt)}f5p^uT4vpNX2q;>GO>%i|XTK)D> zF(@MxAJC$8&yBzkUW;jl;9rE>AcDTv$IE;RE4HSeAKoRk8nf(gu6L#SfQas6lMlb< z*uxelMEc!60OsbWzRa6XhKic(x!&3JT>K5b{VEu)AW17fH49lnO1pX|(xc0jC^$Hm zMazAdm=AcW^i%=Rf@1>L3;L0DO~#XGKhE^iW`V^R2o5g2#6r;UuLBSEZZT9?G-h`5 zhYJcUW)^X*hwoM^*p2Q_iBawN`hU2M*!~E*PW4sseWnGvnCKOQxUOMYqoJ=O$7A*vl+VIy=wwp8TqhL0U z#=Arez(~pOLjgf2&1EePtBC;@ShE$*!beUCndVfm=aSeC$4flcoOiU?SlB%iu8x<9YMbctV^1DrGc#{%7T=Jq+rC{S%7YdIUfxTZ27;(5cb z5jZ12%z{kvm((;n+uBYBy?Xd?{DzUfDC1hO2n%Us0K6(jSVPqIHev-2@;VYf@eja3 zau1ZOk*B*n3H;nz>9Kz8p}7~PCR<66F>-)Je~lE?T!Ndjwx8wJHDTy zI83j0dkglC~|urn4zgAYp!BTcWIIE5Vf>%(u~6CfS|n~PYrSN z-h+Fg3_2`MD`|>%M5!;;$rxR#w#>XpZgu|IZ2Mb(Y~bhIb8@HNQpnzNjDYCv8BHc& zkEA0}4E8(FN`_TyX_$4<4bL5({VXI^U8ou7%2QXqr^n!OVoc&A%H6n{UaDQ`N-z5p zoihGmrztnBtJ##ZcGY6I%NK0kQi-uE$Nr-Zv_Q3h&Vi|Popv5EXqGHY511hUhn{Ws zcZM08is7{Rc>(zRC$)o%(-PS8rX9qD-6qTqL%%GR=w?5Rqie5`6?2D}BA>WdfO`X>0zYN!pXr^bXWcAdFPu=={!IRkk&oIund5Z%isVL*J&&rUgoRjmoHa@GHk zQbImq-w5C)SBpSl>Gq6nOZB!viHGg{q;sF&9+M@pfYeHD0U^BUF|ep!PyWbkP?erZ zVVy;2!3!i-MG%sL?CE&l?`j8v;IvKcyy+j+#8hBRtNcPJK<(QgK@IP6G@44fC2*-rZ}{1HE2qzUKln$ zMeEP5kixcvurXi?dd5Amj|@phaKr!c9?KCI7nNFFT;YGW%@1G^mNEQk<@Q=+{KxhZ zm^$!$Y*?FZ88-{OzXe9FYg}>FYnywVjlR~Gt$&A``DxMqy7)N3E!(X-ud57nfcDN% znyj%>xWBId28jCx1pHmZre%z=-AuHX?oH74eN@KACZ>+vJeAQVl-&&AF&Z3YActgZ z3TK3DHI7YPT;TVsUWr3+N=(~VNfu~qh$aWjpgY@wUA23uZ za>=z4N96NZ;3@3M!|@vWCFEWuByLR=FBG_lg(8BwO^{7l7>^|K znV9E#BTdS?K)FGGK0;h9j7Rp!$woKstJn(iQ{l+&&lOT<(h~h0a^qnl$zdC}M176pn!)89`xdD++6LAUerYVW>Pl1W=<%#*%Sw zEPC?I&s`QrU;Z+og`NNfW-$ILE5Pp*Gz~#~)*;1%zPimGTcld(S>Tws`Jg1VlE0hs zHQ?-K9krUN9gkNcn;>4M#Dsznb46qeIvIfv7jg>k3#E5|!9D0e_ zR1jpSW%XG!=dVm*RGt~-Oq%NHhaz4C=Q!6Kyxk${k4P%(9$uk$7_#JI;Xhj4H)wVZduLT$mlz;oHWhs2Q+^rFtVF4@9CDRW z#-xfO|5&RoS>Z)=q0%DB?DhfzZd@A@#>L_Oy9UFILN>$e?V7Wh=yjvZNE+{Hc5>Uv}&9 zne9MCPov}??8;-uGfV-xC78s1nN*PZ%tH;&M0K=uFWJrE2bhH+j$RoyZbdGTW`8MG z*#d;(&G4RA)Xu!#{QcX-EmB4z6Jh)Msu4Qs41k?#>4eBcHqF2?TZ_e%;6YL*Z5I;< z@N&~~5ZIqsDi@=Kha9bojD&ECcS23tEk#s1o}Dc;5z2|?mI;q@RkJrJoMjhH7UtuS%<093NtQWqsUi{II zkVf&RMD0U9fyu7g5e{(mc3+jkC;^7)BjYzt5zL z4*AAiJiPdaR*ayDVy<b4cXuTN#$ZVi9>jau36i;J238( zlgCrPE3kIFFHoVkzi-PvQP8|?d%hPOtbT(StyVdj)>1Gp6@_ld24fHx{y!2%LiN4k&f z5jiu?IcyD_D9{{{SL%-Uww|lxh^RenR z`214Oe6&eMjhvGtSzJ=RauSXcDELnC*G3f@9QZi?5#1u$0MPBu_^-9>5shrBD$HiV zi)LKI!=Hqv{xMEM)9~{z!aXOUsqdMDrb%e(fBr>|{UkJXpUm2dMvF~5!p4sL6p|UN ziTO~169=$y$*ymC^#hZ9k%ra&8bK0VU0O32Fo|mB;gW_26%`S3dJjrb_`ld+Ya!CIe;^z%Vs<%&Dtoj{tEP#TiW55q}u&$u~Tg}C>HIAl0N!0<$7vQ9RE3*ER(v4(l+mbb)5t*r2UzOrH%c!&+5>b*u zh-yx=+y2BYC3dmJ=e*1bT~b~u%$mCtIk~tT4~vca$i>^Ix$wTg-Os2ZMpP<^Rls;+iYHaN7)SG1g^QgE7K44pDqF-VSA={NgB2}4b;XMP zBi?$e!tX4(tqk9WgzXpFMxLlLJb9Ap@FZ3tp2QAMPt@QJX>H`-mUI{)aqn2SVL(Sk z5Dp2>dre!?jF-zqdGf|@-H8RmA3PD#1>_AVBi%SXn|Qb#A814F>M9d!%x!_#y8&|; zp=2%@h0Lvc+A9xELhK~OCeY{?EyUgp7Z~qm+%|>9eZgQyOLyqc;~2|uaJDEI-l5@; z<}PvtR0tLMk@DZ)9@0P}+RnkPM2`6@H%(uV1Fbd9R<@cc{sM(Kdl!K6QrnymsTQ7HHqUOPi_#e|G=+%P!f&EDXQbzsWmWc1xqnQi$aoL8s(QM(l*;B8u18V*qFmTg+-Fk5m7%ETPU zYLqw@gtcn^(`LE17QPmwr2VF=U@=(6(lml0?L*jb6{fCnY6QYuBO?u6kbU))ymCoz z{$A6fi@t9E6fEi#($ebsURt;H-XF&OC$KzsNoU1csGYHgBIw&9mBw)8;HL z2jOU9_U2|oUb9S;c?uZg&GtLnSwKcg6FCSBHs5n_mp8}Zxdj%nyHhM2PiYQqr~KeB zaz%siWd-0D9%pV&p6v?ChQZeLegP|sIzQw!sa?0Eb=y3YRfB{<)xN7Mi|lg|J&>}N+est zIl?ot$0>>?ob9IF4G3Jl!d5r$EUYOGj?sxj0#wqtJzFK&Jy61yJ=w0UY<&STuZp$x1lU zSg4&yrs@K0T-Ff+h!$2?*++=v~xhWi`$iF}T;q2BbiJU;49=5u6G z+?Wl81FDvrnHkf%)-^4P4Pg&iQ7eA8&q*A%Ce7Mgqvy9?(1LwrAuMr~AeAHuB(91! z${xESNs278Er$4F$L^G!7}C`52F>oH2^C#k*sMz4R28uZkIwA0>9;InaWM<9Rc?Im zMCt5kW91Ecuig|?X`X#ki0szPIOVoEI{Wc@=e^DhOLF$(HF?d|Z43~`c?5kc9VNH* zoD~e9*WmiL{0Xo(pvothdr7vR4+; z)tmHqGtg}#YI0A5g;ljc{%}HWc=a`TTB|hJ?Jl=9g}Ad3uXRO>ViPU2_XbvkQ$c7$u8Wl zIYsUz(v=Q6J1`83Wq07k!i5r`Nx(y`b=%bJEZe_flA*2A?y_%%~A- zXb$pQ`|}9&Nfmka>RICjFE+>ij9W@9gls+)E_d|oxbL~Dq zTho7tlDzrm+79EzCm~RP6@}x)XASiNg7RRX4gE(gnGrn_B$+eUc3XNeeDR+$M>K-?$_kKDA_R_RANr5{pXKH zArG*fl$bQGZo}oEcf7QIm;$6=!tSIAd;_{r^`K%uaz!zmK_ z^yvwNX*7v%!cL@yUiY@jf)D>GG zrcxImh|pTESgASI5Dt*clB1V$78vj&kSeWGP`sMX=NaaY&l6PA8Nt?riI@2heY>TEgVK7nZ`NnRYeI%E=s4obxo>1VGy zy%OBIdZ@dYZGDQiv{J|dM^5M>p6=mKdJm8Fgt@So zyjgFa8>!E_aoeaF+~l7j*Lh0sR-{JlKy}*g<=gxjY)>{?uM$;V4Cl5$1`-Z@&&$#Z zPh(tJEfZOboRK-#E^W~W0+~&}fIBm=3+7E0iGR~#KEj;od&Wxg5@e5;fBfSgBdPP2 z8iKt1>dP-r$ZIYEDxX7INE1L=x!}K2XhI=J1}^HLK>#zo&7EMwh}V3%g3)q-&s!8c z)9@90$T9adI8KOivw43u{sd@wObUL>$U=Q5MP0Qp$Qb?>=rN-ur2fDF+Er0Zf0O`* zh*4-e)zp(tjtQgL$|&43T>6UgCU)*M)qR2d&G;dnyUXk|mU{=x1t3;Z{5LK(FM}a5 z9xH>mLs%IEw=O-f+*`JRwJ}#EK*xqx+a=TWq}f8Ps~i#lcSjo(H>~#EMq8TQt#HrO z~Q_3cjt*N{X z&fO)KpFlLYY7*Zti}8*%I7EKFM0lJ>Cnrb99)u)HQ)y4M`TOuy7PWh5 zv$elzC(4^9fTwY>)0n3rDk$LGxow)&BAd*E>mJf%e-)TAcQ?=CQDktmvYG=z5T4HR@4<= z8ptF|P}rci)c6hjv=v)RZ_|4NgDK$n!*Xn+Yo3@50QU#5rj4kjHL$xxFQu*){GLq zdZ&0sB{l1iYkcmknIk+aKsUU0R#9%+qy-}5R)KkWDz z(Lx#03ASyQL@_|4zC(s?L=k{Y7eM+3nd!PeD$I`7o(};TAqBHlE6hT8W4EbfSS2TC zAIi%Adr)aF3bmb^?3go3YBurA?jCM7R-6`mt7xQGGp$P|Mb-k-j!GheFyi<+6cln@ zdHJ4@`3)O-cq>ij-cM@o9qDknwz?1=MwdC>%^h{DKCZGxd^V4Af#BEAw3d8c>(K5R zSNb)P^gT(X?N~r`V9N55nN=_|?<_=(gw4tFIF{|Ny@cWZ#Xk^FqlF=LFu(;u3iA|H58*ouYu>7s6^0sf)8A;v@8YkW_3&oI%kDCMx71Hof!e7 zq8Z!5SC5(DQm1d8Rziul9WJ>A@@BSTYYPAT)=;Cj5Jum_6)|FgW|-K_6PWsC$EawQzAa50^*pps>uZ5yiS#`1G*irZ1g`5v;;JRt z?#Sc>n4AEg(h1&5I(gq4}daZ|;Ix!_W1xVT&W*q`g#jkR?##Qh2|wu2#Vy4?A1_qur7 z>$b_kw-aq@@O%yTYIIsH)`o?AF~`FJiIC^`$CIq8k8OjKHCqe0iL(gTiFvuxtL*{H zV`a6JtOV4JC%M%ddW18{+Gi_LxUrLib*8yPuqZD^OUFq;JjVmP$;vx&N2Q9*l#IQ| zbAHFwP)X##L|dDOS9lZom)%)Ole9IE+Fi9JyS#dK=n1i4S5nT?vF6oj- zq%>9c+fl(I&4Kwb9F&nxz{LBSCk5}nv2D8%gqp1@A*tMu-0I^SXT_j$xcoaLK6;st zkFH7rGs(H!H-3N3a$c`fts_4|(JWhv%M}H{E}D{uG`kJey@pO5$>4hRQj{+l-8tgqk*2=hA)Va0-yc%yvty}<#Wl%`HGe$ zE4(Xnt61WYLD?^%7^$FmfJzU)j4Y^6S=#0oQXa6VlM4g37L#w!9prpbm!z!Ma~3#F zpRSbMr%&v9Tm}{r`NtpMCs%yvTcdj+i~OGF7^0)dbJO9APhW)L{+dZ;6+RBZV|P};9&qcX++9uM?i~h&m3ihCsB?)8d7Bi*GAS-fmY+6x!C1{3 zqTN^qpy)ZNih9WhtkxebwyllZF_LG>AMdy4hW5-(U!z z`@WxN6|BOY9?Itj>;-Q`(@OP(8Q~n8nM`TipJiO%)5TmZ3X$C=RTB9YTW)E}_K(}8 zM2rqRFx`I5*q$Npx`B`a+r8QRQVi{IWs5D&8K4>0}WNhY@Go{98zuOq7b}%=Vs}wa5Z6OljqM@rxfmBv%PAO}FOU zAXAeXYiYN7KxobjZxh*`p3D}s%-Cg9);qC8_0f0dZ@Mpa55RZlZ~i5KHkKJ{OP=Rk z-A*VQzXFtvomKy(+t4P6jn5cj<0J7Xa!^BOm5-1G9FBxHeHpgAImr%p?6{`krhAmMBn{L?& zxi)_gibEU$0&X|R*@MfDR>BkV`pxCloAa}qH?O}Yu7Bu)sFnb z)!WB?8qZbYhtI`&%{5t3wQAD(+GvF4RPR5c7@uoNwHL3$gz==h@kige##;(o^XHxF z@gWYG5wNF`@V`ax=KojecCm}t&4KUCG>OM~XPp>BL0>lj=+Br3nFvMk#QM^9Mgbyl!ZBrpEqP76Tc$2jG z!xfWjt_&;bmCOR&e#CwB7;&vReM92hP?%|i!5J=;2#^gxW3Yug?5&b)#vZr=TLt7U z2-)7=LmoE|LBpA#oWTm;1^nbr#8LodA0KB9M1*kD0X$VNs30bRnNg|z1OO7c5i zuwj@bz2fE40}lqMV`kH^*v=fx`BmDBkQ?Y0PQm-{Z%D~ZXDhBAtvr}v1Y(V6vl-Y^ zp8vOzOY`Cd`H|VL8}Zzn_)L@gl>m*pR(4*ob#wszW$xZRHSZg-5HGtEoEsK5PCc zey&FtPj5CAMm;pFzy^YBp{S3lX#J6A994|>vdmVrI;-{SHCLIqW64U5D;s7_>x4TZVlU+UuV!{bTJ3&Ok5WRq~cAjS-mG7XW2J zV06fnSCWfMD9c&>MDtvJjrW-=Mj}Cb6Ac(%0(#POy{UKxs@l!@CG?@lpT5HC3BUj1 ztCxYr{8*9dBkdg@(cknaL1AocBnT?kEzX)ftGKJoay2|VRh*gP`#x#iK3qEF2m5mp zWFnVaOoRlXF!y019;*lmOk+xH^= z^BRqW9al_~r!vIE1)-$3&nBLMTJ;v{0VbF}#;jmVhM8N;_D}ei%nW94h6rzu37$K5 zFKp-f4Uj1;ukRAj0ftw?Lg zM+g32_Po|T-`L_DQ&Wmz^FY%D#&Pz+?G6I#?s@e7Rg+W?RuUp5lq641hq-udg*lh< zqU`1%?e6gQL14bY82rayr}=)Umu5llg-lOMzwErQ6`^2%+s+vq67fHmQ*sUNR&OeO z-V}Fs0!!R?sFIFOpwFH|g-0iTWI{LI;SSTtODZ+NIaWYvY=ZNnB>9m@yFb|GR^d16 z)3OiR9HCe!)+dF-Z4fLIcmmcZUkC|ht2m}FwQ$`$;SyP?o>9)PA7Tp(6Z|Qu~1($CfDeN)QKnO)gH{79aiyP8_=p6E}h@ zu{9Za`tMj5R}KCXgpbdh@JlYn%b&?WKFkmH!gNj>)CBKuJ#Tj`TQi1tYakr>hEIPj$HXAH1J`=momv`-4yMEj&;PmkHo z4N7$cg?dDvy?E6vY*z=5dPKKDVjb=WnMdtB|;a2-CBS{DClZ=PsHxETNwNE_A z2A|J4)CQli0XIpV#)ES38MUk8SlR6}8Z|pmEW-BLiyY$wI5G;UaC>OUG8&nb+DWNRe0~K=Z466sqz>9CiZ+C* zIHJ#8{KY|h#W8B_AQEF2X>p`r5XEFn#9Z`x9nW^h;xM+)ctplAN6E-b>DY`TdhVh) z#!(u_$VYqPIkwM!a@lB%$M#7>dmJf8_}Gy9<36^}p2F1#EJ*LAL{!L;B(Y~9j1M`o z&ong2_Sut+V-P0W=df6lkDSCll4T+i<@U%%2$o|FmY?1XncOi3?i|2OWbqEH_y6;; z0DVO7zRl>2)L3kvF|Pe|>%-*Qe;U{RwZ-a~{QAAKnf&_gGx_x=6Ubx&*?lIzerH;f zUw`uJPk#N-b5Os&70=PBpS<}#6Q90C$vVq2QJ4Bg+_I=^FG*?J=`}jRr_=CgopYl} zjb80D`=V2F%neiKEN^;DHTjyCwV}cZIU|x~yy6UD@G1Tsw}XTCNEGxHD;5?Qq4a>Y zJl({yemC5tel1ghVWBu6T$O%@I40SDTZtoW_Bit!ZzR6a3rU-T3$`;zT5(&k;Lsd` zJycx6w!UVWC_^~N?bF#jZ?=1IyU5y?6V=$fmJROl9W8jy^`>#1TZlV~=gbm1soOKv zt}>xjB?Zk^OnLK08Q5l4CiUs^UJyJI1Yqdb$VtV9NTFTWh#h#E`q`Nrsy#R;`~-?5 zT5cfzB+Z2C4FxH{7ykcIL=?FzZuF;@X{sNSnLX2>-taY8I13{qnyaDvfwWED4&sY1 z$x75xo%lJBui$T}pQn1sj9{rr!Kl*Y)l0JGWo<-*gm*pB{HrgMk6Wrxo+^&-20qwR zRbkO{E0tTW63e>vz7K#0TYgA;75OVyYH?s*lPcXNNvnzXep0iT5F1M&wGNKnUn2wf zdoAW0osv`jU)ZvnOz!^JUp#7H!W}G+wsur+>$vo>G}?4x-XLh+P8)P)+AL#;IsCds ziz@~88(FeL=3ECWG&wrE{J{|OJv=T%iaadYk}Ev|f(M&-x0B-K5+M`Tb)k7xG&2v( zx9Q5>b-u}%dbUZ5MGLURUc_ziy!k#WHCF1E<}$6Qqzz#Klw7^Ac4?Xg!{2P}8*_6> zS~TSH9k`F#Y?_s{40wUMW))~m#J!6owMa}`yDXaW{0+@+S(yWz9+D*Alyq%tm^Pqe zEZ^#RFtgk|Xl2BdX;+9Lf&`vGpIRjs`TC%YED+`q3Vo`Q-(}2-O@S$yk3VMccY(Se zJHxj%Ke)H^!FbuYxuD`7Eeu--=|3*9err?AH;)DEiepsa(HQtSP>&&U*U5QGorI&S zZ(et~(iP2xBHt7uyCts~=mGwHH$aK)5{i$+u9EZo@z?@WJ^Wu#J$yM25#X0m4S!9S z>2NAWV~3*#uema}4k5wVP z5+b#Nw72EWTXx5L3oqwm8O+Dt1ivr<;wK=_VGGM2UAq@R4fT9a-8%BN;{b(xHbX8J zD30eWC*qDt$#bSi!%ck6mlp>D2;7UNQyKhES_XfFn&dGr^VTl&qWPMaBg?veT=o?$ zXK&RxEkx&aJR6%>@m~B9o->NhJlPiZ>;l7a@0Am+=>LX*L~$w@!1e%zw%C^;fq3KUie72{DIAj|Ftxoba=0uWHx0Py_q@N{ z!Z~1U<=skQCDhX4QA5X#G6JdLri0DkWIwd%H+Fk#0gN+sCyiM+oZUriFdD3*`3Y_R zR!SQ9wYm3(ejk<@V-k5FiKk780=Ftr)dd~&TkZf~JOTm+zrL?aV0;zk zaFRi{AamG)?@BeN6=aT=7}0{f{OZduhl)23mS_%2sG;J`p%Tr(PJ_jphf)MW&br~^ z&BNlr;i@-BY^!7Q`TShHkcBIr#n+$T7U(mOyBE&O3-#HPv6q;Q7tx*CK!-!+;El?> z^RwzABMs-<4dl`V`s~lLi)YXUy6oiEMKUeMRTGcr$R%aJrAA^WWV$^vzb!fFE+v<3 z%;nPNrA0a=;k>n>Y_w>v{n=`v9JFYc;cH~S%(LWtGw;fXcAA`JHYT?$J$N-Ga5X)9 z%*zRjPCT45)}INMw3Bu-#lrbt(N0tH!x9f~O0;Y&11uUzdyw$hlq{q1xkb(G66EsX zl%jNF-WZ%d-$rFs4=3PafKgktFepLP15VOIDIX+dv&9rM{GLbwwzam1@2|JqCC#5F zFUc<)6K*o@?@qEc?yF7-C1;dC*=Zpp5p!ESHX^IH2j=YZ!szpmF-8k@LVEJ|tr!&3 ze_9QPYR%S_4sC_Ww*N_O`=bq#KQu2i(8%?gmNRqOz{>9#TLUxL=F}&|MKc3I!E^(O z3>}Pi60#Ar=)H%r&_Z9Mse+gB*7?lDc1&%M1;JDs3b~``-Sr)fU9VUydjAr#Z`${} zIt(f?v?#xBBjkpeVTvH+qHv?=TWhjE1>i~AXh}iAA5;TrU5nRD`Y1V78&6>RGAx?T?_$<-*xfX6%6n z(+PKI<`&6S?R+x9>rPeXR!oK*{lRbF6~OMU;iq33;OHcR^*xQdu;AGy8Emt46Pnk^ zH&n5lyrU&uqVD_r`tqB1FU%4~%o`W4`@D>|UU^A#1}uEzZT-u(iIb{M~{C5aNKz7TDa~ z^PqXm=!(&t6k=&%@UKf+RXu@Ra0M?43b_ObYj8(-0iE;Gm5YbquuCb{Os`n2$Y1_X z^2N!kS0^vGI~>wJdWoSl)pduHc7^G8hJQ(hnfP<44UyZI++@8mWksztlc;h*GHQ2l z?}aA3%!)b(Zv!h`?FLAgkYe2}LUD}LXnttF)amtmV6Q@g9)ndpdgc8k*QR5bFFoaokG)sl=XVfB*X1}U`^fkGA)&4(N$eAs;4nJU7E|z~yw5(A~u)?4H%fJ0%9!bk=#pKK;Ch&4O z%Vd6<$$Yk^C0{V5;r~?4){}L?)X9dfi_~wteEIU_S6_Y!|9$!L<@SI7`09^;__tSI zzWn_cUw!q5-+%Fke|!1G?_a(8{lAfyPdjydYNe_Cx0jFp*7oFnA9ITDxe^J>mCef67LUglqu^IB=KzOvQfU$X@-xna7$Su;&@s;N289?};w29VH1 zQ7}1Mvhu`8bn}`QIg{{(hB3T6`Qqfu-`GM2T$DF_&6K9=>T6QgPyiem#r8~k=ldJO zip=-191ZDjWDP|)PMl+V6I|6+CoH(qf9v`2t+N$XMJ;KuGcsgfyj<1=mD|7mhG5{1 zykj5A+fv+@07Ig_ZsMoGAo+D`tupK#>YxF2aLl4&<=N%M55K<#EHTH4x3TlRL0ccG zBD63~d z>|B{fiZ1Pyazd^_^H46p6XbgdDv(%~{2xuvO=d+wHQO@onafv}v_O~0aTBg&%ly@) z*Bg3GhV&J8U|J<*PsOBx$2i)#e7BT&*sp_hHygH7bOfVr8uGeIGgoh}Z#-m*)mDWz zq_$^2Yzc1&1?DXq1>}@JR_0h!46=y|&bG=?sCo{FHVvAM7ANE!*an!pQ3N?57bQ8T zYgU}o0l)Ym$!;jDW`?A3!u#AVARZCocu?;OS%8(Ssd4D!$0hIQ<($V zWmG5Rz0mAya!IA>>58dJl!{$zNi|z;8q184hxV%IT7PS%P0=zUgCw z>V%LV4EOD`&7(sV{3jR?$BG=3*`uR-D$AoIA|yFFqH`hj(a}k7+O}u(&@jRo7xrHr zgNJp0t}9WR^LT2gVs>za1er-okzLnh*dPzQ@NOt(Ct^+KtULRcmV@KQ?XqM+3W3nz2qgE5c z(vl*^*h3Y>qN3&I`yl+L8eX1`?n&-P3XYyY{%-rr&SE&EUR#~u66*2uQ(KiywH zp8cmg=*9}uQpDPR6th|yR_#5R3_p|MXEOYJM8l6gsMZr~egumG_$o=xRVMD(RvF%o zts=!BpuV$bVQPpHs!NvwA2zt9!z}agC8ChBviaU-*Rq8HB(NXgKiA?J|NryS=^V zo9Oo@PVv;$#ela@f7>kAzU2ohs?>c@_1j75>!v(8WEOk9__@^(d9aTs_t zbx8EsEQM$`Zf`DbH@&L?a0cdgLQ=5$UFs{(T`#vRP%Mn3YxGmz#eQ^4!$KRMshEOE zlZbzohI|%J4GkxTCKjR$@3W7>7u+x=zA*8Hi7$L|z7T0JXH+r?Ac)`cA|-3(TU!Z0 zGMd}hE&h}_@E{d}{sJ=bKw@WPmAye-r&gJ|ZjSfbBC!%;k5oS{4wE-kGRJ^Bc6C* zewaX~r>NraQHnUkQd@?2UWA>kL@DIkfak75j=#38NwyL~Spyhqs7K00fQu4v7@&0I z(E4Lu6&eEYwGitYCJoG>v8yJ}qJ6^|bK8i2KW2ofiVdS~0hVCflUKG1wf!7`aQB;7 zICx>JR46tjRNFdQ)@I-gigV@$w9oZ|on0|lCSER6i2Tlf=NvS8xDjYo6;K9BVHI{V zXA9&!OQwcp-tp_7Zo;4|3q z&woa8{v>gnKcTrp{pIJGYJp4qye8iqGJHv-Sy|dYouUbO%i6MKd+8tQ5C=|*5Xp2; zYTMvyo*_lFaA`4Q4jGT4B$6%J116jBav9bxyt=kWl2rxG0-!^QLiJkYby1I~(1z{7 zf=_Ro8+b)kQ#8sNu7TLW`B?jXpiL^kYg!Bn&^mdhR}9=k5DH;)6Y0|EyN)}7W&J@* zn#F*V8>Bv1=dBXON6$}w_u|p(X!paKg|H2G)5P36c8J{xQoB2C-@z$z)uwzoWMR2W zK_q}WG1O%b!5%YO=~^-yDG5P)O^QsHfdBsL(d4s$mEi!QEzQHeL4Y`UQ#Q{CmsB%M zYQnP+oG0>V=gZ2ydHrXwLRs3p;QJxbsAp<7TZdRVy}*T2O0gzTkReuS&5A^(M=t$3 zpdBJ%Zf_UjAlPlQTGl-XhpO4UTUZIZyD-={#)>kx>5s`RW40z>E=A=M9YZSfXAJTM+LD#a4a2K;xX}ok)4t}k>MTES$0=4l>GJP=8}B4dW#cbn(YVAGhAr0 ztE!lUSyRKqc7NETnkYe2^*H(Iqs!dP~?6L}QeUA2yih|MeHI{^Jj}nzynv9#r=P zewXN#6!mh|rNv(JkfXG!qjZJJne7JGaIT=Pt-+@#74Ka(mq zsd7WJnp8Qj&7{ipQtc^KIS5Hk@qmLCgRT8q!C?tCGJX`)Z}^&tIz^qaEeTy{CcPE6 zN)?Ln`?*J2O=du?9nJ(y$pK8r{_9cPq*pG;mTeza-v-s4lR@>Z8dTi&NJ7+wLK#10 zA>UqiV__jVeWa5I`*ld;YkPzLw$^D~k$*(r_<-oj9rN7}duMvyJ| z@*S;41oWJ{cl@E|RMV|7274s&8KlpGq#?-+W69u^97CfR0Vb0YVYV}%&nTk8J!2y) z61t4XqLFcwPK!w+DKQ!C*2b`mgz0#J8%g15BsoNlBqE!{u}u;)o5R?G?&*vE3NwPw z=cqqEB$ezJmP#h(@h{Ijb^^K*=JPMYIT9LWbU4nH^p|)R9RbJ&fOgHGhU)}tUX0~L z30o%%)+X7K=wy#bhxvavE5P@P3rv%^xAjco-o&wEolfH3B<@Y(-o!k-XA<{5mbk~H z-c#V)5s!8xzp#XZGT}i*K3*!8)skKRZ}#53Ida_C6aD{w3Pjrx>b|$C)NOmcXL}vH zAxjz`h5MmxX=c{<5i=+fRfQuKK@(u9Yu5YSZy*x@$sz%QrBZj>L!5{+Z55fsBaujC z{xb7{7eBC)ucFKPaS4(7CO|=t3G*AUM!u4#72$he`=?E`mXzPfJU_5~DDbQ^n(lej zH?A+HUQcnd-`1%##mNqi10g%DH1el5sbvCuYXx^FJLUn{Q{HyPq0bs(b;yGo~irPEK58#M*Da%A?Tz|eieM(pN0zU3XYqrENa=Uh6b zf|YaQ`p@78kvAOXzF0|%7KNfYuGVm(Prq)trVwn;@Aa2QZuHKc9EQ?TVPK3w(Jkqm60!;D|vj+kt6ZXWf5+N>;|qX%(c zGeY&biC8F_6Pu3RGrWH&9B!Q%g zt$9MWtXzlsS!g-sU%EkzV$`i+2%mUt>V|oEc?UG~P_ICI`|+^xaaaTM(>*hcY5~%B zOAF1}|4WNgV3hw&3d&2lAQzBg)>FQ=`Mj}wG)Zf!ToOI!JDM?*zc42(1Hs=*Qmi^B z+KfDIgk;7T!7|7{`r?7@U(8xNkMMp``r;;kfrmtAAGqvaP}JiE{PtqeIRtyj1`o!; z%T3QuDporjs~VadnqNyj?5{ZJa!I!SYR|_1BRF4E&im;6?^xoctTI-PEB-f-|NYJD z-~aJtS);DaHGg;(bc-4(&84-p@KY-kSxZmuy3KT)1$P=H|48 z`CGI4&r>3Q1*byC3e!%&wt~syE6&oHjkh_Iu%*)~)6AQQT;E%Hj}a_n1J=fN_r^!EZwF!)#k+s%2m{deoTiuCA z#@8DqXM``<0+yfAWYhTEmazbij%Jy2cQ<&cX|jFkgt8HoGk0S;eO1JY0X7{+a7N7iU%HdqRJ6=fuy^EZ@&YI6g*|{}we?$$a57;2 zOgOg*xh=8mBTzgvD*V?VGiYKjlnYIQDTWvFWj&`>O zr#04Cv2yE1NV8Bb1LJtW{_|~-@q{bGdE_;D39QG}nS^aK7saauxv;9RuMKx97w*gE z6S|{P1cA5)P5zcKw4S$pNe3G@lB(8RHNN z4$-=T+Cv>QZ?)TU5AVCYvcuTTh@kV{?s~h~_7g=ACT(9*jMpfN>R8WGyTz|OwkYT> zHV+J<*ZCAll2|8I&=u8gBe0~{hTT-{c9=9IhpA!k>+9Uetthgb9@G#yky_g6GUubZsL85ZBoD5qICw|KkxyY?5rqrGc= zBz{BIo!QVGC}H5Cv1--KAdR%%9!(Ug11MG&B~w_1i{=&-ao`1vra9;>9P$aaTl9f~ zZS-b|%WhJ)IJh{rVbHjJF2ar6?(SfOO7r2Ev4O&2?1x`j*Dxs4N!OTk4NseN4cjJN zW70JyU1QQU{)u#rNTO&@8DZA>5$#-ub{w13v}+w47?D;JEY9~HOhg9kRMLFS$ea0_ z*RR8z)c_#>{&x{`GMqVdIHaVuhy@Ta(6=igpqR2MBOrecU%eH!5OP_ zsfD{DTc!wAEz>67GUq`;HwZc!y%eo)B(0%Uns*kFJ@$J$nobdtFp5zvOXhklwz`*4 zLvV=E!GQ?R?Hyt(1ZF4M=keKu`T#mL<#@V!&l{=D8EIQ?O1Np)a`hp z-28!)mn-y#`F2Z;!g{zHV(qoZ4*j=Lg|~&=1;g zNCz!|{%f$mUkS6Xnb0r`gc;TgZ|i`eJCdch?a>6JH{Ri}z*L&D6mldX>j|>rCX>Md zx0~^M$wuPn+npTYx@dY=S_1kIRX=bRy7hZbR6X3he3{$^25OTT{k>x4o`@YQ>zj#- z!Hl_tz}dqkNvK3YJ*;YtyPb~rgLDQPkzOiNd+Sd^)thiClA<^JMz@VpPRFFKioX_k zQe5?zJ>qFTHR)BijWvIeaV8KlDpl49qNI~nYlKCF9CsfXZy6yYQf`eJd?!6uuc>4d z$%uqqk!iWQ?WoBfXDuOPl6j4q7*FD_L;J}nnsKSXMw5(44tAPucG887A{&!hY@9KI zj5kP-WBGEdVS3NqG6LB<*?$msD7q&Vxg_@rmPu8mm1v5C=Q&(U`Jw1HG< zkwga)s`VyMizFSLVhz)&MN%D+bS;wNxcsEuc7#zb>ft&`^2pmnBfGM-ugFfW=oo9G z)u8UWdJuW(!p5;C^(+5Xx-p~{5)D-TEu*<4FRbiKXH=W(C=BSt* z{J3FdYouJ-wD^QdQdT+Cix9an>qU{&Vg?fm88Zp0{V-~*E#_7!66#2-0TZ%38O)@} z%_h=+apfT}b9wB{ayP~WxrR;9>>QV{k~GW2V~q5P=IYowG>y)j!euZTX8(9%nEiq2 z>7N8;A8FK^p!O5gz5%s|mr(KzWA6Pt16cf6%su&W!(c}R3~YLdwpFHhQ9G<9b}hr} zBLbD=fps<%5kQm8;WmNqfJ0k?L^f^~nq>xCA52iaWX4y;Ph-Nra)}I$(FB|mFZE%W zX?D;@mX&HDpKIs*MXSt?SqI6Q?^s?R!Y}2kS3|HS6{&i36^j7TK9yq@dE5ucIz=Gj znEzAb^!jfBnzeJtd%(l=xFg;zV1Ote9!;Dbbc7q>2p0n3;evH!l>6X|g-?vBND-lh z4Z90s?|;Wa9%E})#9cst#hAyqx)|~9k8d3M7+X&xdce<)fjru%|0SW1F;zLDhd*G% zeI2m~2T^Hh~R+AX@yM>NF=K{)u_*G3SIG{_%oj32c+M^-0^=tiUgzH3^K8RDSv z{m`r)XLJe~-g`~z5o9APos3bZqPa8j!B`Xec=LG#mSBYG+#1FQD4xP>9wiZ@cUqi; zh&GcyhlzZoc|7J?PI|BAA=5bCga=5*o5h>A;i2Kjn#7y;;UUjdlsUX>?2h-Qj538s zachj_HOBn?WTWaJcs*DASO9yIuRSRC-ad?-V{|9c)8}JjV%xSRwr$(CZQGi7V%wbP z7u&XtNwWECchB>j=j^`e?yBm0U*7Ih-%nM|=k8EZQJeniDoVWQR`FLod0JY4B)T$8 ze!75)=u9tMO%oT`_#lOU_@MkYsRg#N(rz+H=waX5t-{OYSD0|) zH*bwX#18{az?k)HQsQs)UQ;0|cV0uD&k;Y(uY2Wk z=#OGop}d*=S&{WmLcb8&XD@;o>T?|&9w*v9*dEhB`H%Rh)c5NNTF&O@0BZHBNlURE z_`8oXcg9*y!2{*khUIi>ROno}yv;zaqe+=;!S^*!UwK$dfVcTQ*I!Ryvs_PAGxLMH zP1$f_I&LO}txTXG?W}n=nnN-Nafbzq2^SwQcNu|p3gT(aAWnx_;!eT?<2={){42`o z6J}=vuZ*V6^Wp5}=;dGGKucP^+c}~6+PZoM@_uodAqiLm#!e?yZE5LpJ!E8{bzo9I z8N<(~va+!jiC016S+4o&eDhgh#~HhIAv?XE0_}1w7gA`|VDIz-bbZLSl46xYeZ2iX z73efGS%wdSqVH8m?|6_r2VlsihvQPYr|_o%MoTJ-xHl&G1Q9|Rts#;EXoT$>vo3!{4YV&)`sb4EYEJ%nDJ zl(!GM&Oo*#)wMn8*~;nI+ouVc#NcBq_(Do=Khn|e&;=t#7O*su^nN|a5i=Z>qh(0# z04vxJU5i*2E1Mf^#&Qg{ZnqW-3jIto*GVuwN6O6#pMYPsCN_Wz&{hf z^wH1iDJrC{PS;%{dY-ua32r~CWA?eo)9pA`uucp){xTdg_T9Hl800KXseSt3v8%DP zPCNInZ@R6huOVf1j|M^JM2TlFEgdWinUS*9sPPCQ?xGw%=EVle{=?nINBEO=HDFLj z{yoX|B<@*9K|mkQ%}o%SB{^%fJ0-vzDt!CWBBL-8ehsC)ewP!os_ZNO$du4JZ1s@z zLyp=>%q_=jw}9|9xne=6>GsfXOkDbD>5ScZ#?~*)?=KuugfmZVOH)QVEefmY;WX@> zwSq~qSsJwl8+nluQh9&}ief;We^cod-JxBY;@9sep%h&_Rd<<6RUv<^ zD6G}!s2P!He{0U;IeL%Kyj|0=6VL(}7vJGl-Bd9GltQI%XcIbmGY(P@$yhzGvcYi9 z>#V@Oqc$G(Hp(=0(A-;BYiHmbzsax_jBYuv($Tw8Kjj?87DtS_UMJQ2W`|_InAmg1 zr=!H$?MQS-zo&N&Himg|XcF)D4D1~!9hZg7k%W8(X|M0=;BZ+!pH5frldd0IeB(4Y zKshZ*YKan?CEMIQtKO{(^%Z>~BpZ#A;e`%yHsmNn<_B)nYFtTUX+t8TH~5jKB9%2C zhj>7^S!z*A|M)2&5vhF|?ZYM%HyN6amNgBzNr+)x{9fQOsGp0<4pz`6P!IkX3QLR#xGiVBUT`vwV5s z{fHWu_@ilF#AVviX%04dZl@^!S8RLSbh&rbvGdP#oHSEkt(Aawi<>>X9_Vbop^G-_ z*8YSwp-OxR7~>2feQf9$Qk9S|1M{SF#OM4^xA)tl!H*ra2L;@fUrgGw!mGW4;jM+? zCI-O>_M*KSH+C~8)h7j)tG{l)gRPB%{)gI$hn*3&FH=DD*fFO^>l-3ffTDu0SGnG) zg?IwjO84d0E6BB{nB#6;vLoQySVWPr!x$T8ijc6S+-uR|(ehb?+CKzl ztBp*)qY*jRDL4)HMtxCZ0mWBjM_4g?f&-Ngr4SLSw%dHqOZPecl)qv%dc2HJo52C! z0!9CNFqeemv>zAfCpysMW!o?XgR|d65Q>MHwAQ`(Mt)HnZd=969`$?L%zH%);5l0A zy#is|6{jAX$zi48Si18m_>>)ns7?Du0r83gpBm7N-}yCd1-#>fqls3dVN`VdfjnpTi`D#HG%12w{RmQ2l?(%i zXc1rwrs>o5;}mdN_1J5214T?SY|xD;-q`C|n<;1|y_>&%8roSBcpwHCA@ihdSE7ZK z1s5V8vNGGO^|-m(5*c2OTYI5d2h0jDGpst2q%Xy_L)p%>VAnZ1dJDU8=2PIC{hiVm zt!8Ee%rD{Jx>Q#;x%r9)5a$rxlgt&=(ma@70L*g3Uu%4 z%CV3{+gn>eX4+qG7Tg>@r@klY{q&_$NpJ0I8H!K2F8bTK3AaO)0a!tlvi3eqps@CE zPz&ZWa-ghn4yWVDz5)>@uUJ*Az7pJ)Q~XEk4sq8yX2t!t^iB~`9`=PwE}O7lw|oNr zGalEiCXd)R)#RmMR3eI|DxNaG3^|lQ_YJ{RY>TUPojllWA!U~7irCsmnFpLv6$F;F zgr&oz&jVpp>Y%6kkzRJNQGm5<{^%(j5=eKUlp7#fM77pN@@pEYL{R1 z6&y$Ly-1*wNo$PXg$N$#4xe%9Ts>me-sd*1tDP7R3<@YeIimU+(2Y;;+RnUO$-=0|q>=-P=$c((Gbkt53+;T=G*ey@_+SDi=B`U_;47lfVH@dxQY9 z-AI)Di5#6(ek`|8s__m_nUCfTC|gM1u9CJB4NIP<_4Mg(qpt&1H7yZDPb(pw>*6{3 zBJt`l#>aqLZE6&77Oc?vd-BD^==*Q77Nv*So}~(=zyfE6Z=Pk1^t=FO{cqxgp6%t> zmqW#eVVk3WLh(E9nzi7@?^&a9*atr>^2YuweCnuMl#f+$%@t4N3dET zX01UyJ^>h_#oQh;kt464oy`nC`VP#oq@lO%G%M`ESMz34gKT(eo`EAvsRTR|7s%XK zW_SP^l4>dDM@O?Avy^Ifo_q=U(6*}%ue0f8q(~*@9PsMi_sZQI0)B{6@Jf&wOS)QS z#GVPo<7m?kKtenMNQrPB&HdxR4J4TxAI?{t#x=b()Beg|VE#7*iH;rM?~_SLij!L7 zCu8A+o{8LO#$-)(5O99}KxUnEnk+&`Z!FT-{ig|q2rW23bEktqAY@xwSRp4~e|#eC zSBH4QpvF89I#>GKNG^nU+>sFN&>SH#Kcb4eLQc+kHHQy-Egm32QbH9D4p(1SE8`UR zW%3)agf-EOak}kimLP$)(O2=sywj$j)X{g9)!MN%!4!in^N^?_sLP{Cf_jB0)ikjr z-|sF6%$d@}OOw%CY%Yef!gz!+mE>DCOX1QyacaD>fOvj*o zKu#|rWoL(RU|%@rt2bxb;$9$_M0F?Re-XO1=!O58JG)R@#-1H(7o+n?iEjB|P(@i? zjI7?7Ctaffq$RK9O~>-;fx~fu{z~48?Kf93WjKhLG+TWcCu;YB zVo!E`2C)cv=cSK~RC0_Yl2&t~0* zh7sJfioC7}R}8%C1xQkhScOZU9i|-8QG%~||F;i+Edz{P1{w+u3<7$;j`v;UzWc$- z7f-!y^CKR0XT>SU(k*0F1Shs%@5E2O>(r18r%V=Pwb^Z;wHA(lBEKL&zkMLAakGE8S{^VU2V693<5SD@R0%B zUuy<^_2@@4s420nUFV>$QI-vs@4Dp1k0wf&Jl`|MpyT4vlW*xK$r4G+~= zadS`Etg+36Wt>K_adVvG8dq}<$)%rCL$fa8Zt8tF{Ma-M4Y(n^;eXI;x+5 z*V=?Qd0IaRLT@d|ds^EMT58#ijBOhBeNECtN?LDQyy75B#z_QsQ?kbBKIYn1(*|T9 zpG92fgvT_BOmq>x7l!(UZX`Y)bgm$}n_+{XHw z?nOQPv0w`(pFC#NABh|@1b1s1V+Twve=XUYr8ezpE6BWxQ zHMW@%&m{SiK^#AR?^Fftw9rkm_TYitxZil75|xq8$+ACsmL+^NY&gG11&B&i=JuhA zYx@!34j11B70*tPC5f%Ev;uq6|L7KQ=9s=sq2psb89Hr5jcZQMqgg+;^T=ydd6z>Y z*GQ8#`++Adk_SQhC|Cjd#~E3lHA@@Ew6TC(TW;ps7D(#p$PNKxG&X{lhurHG?Ah8$ z2@NSwaKH3*dzb`?X4vU2>DaAQpDkrjYZ;+F$58%bD|2AMG|H`X@M%IfIAjdqv7$-3Mi8KRR*Hr+lEGIll<`Ht&c%E2i`v5&s^yu z1OVD)0+v>%Y?!m*bMU|U43})O^Gedr$03{%1T#wL2fX@bvCS-v&%@NgT;_Y#_->5? z8QKyVKPJ^-L?ybN4C-n$O?t*VglAvxtUPTIl^dxk<1hY2*MO=jeW$y;|fdZJsJw^CjDV)8Lym7(8O_X&3m5oKL7wKy- z)oX$Tb2{suT$kMst&OQX9Ph!=2{2t)brDj@(lmBGT#3PTYp^ z=JCPr!yRH`K?Kf>GuK@@@bfOOZIiFj;&>(py(%rNh2C}+8(f8x{OQ#VJ#e? zeVFTLtpQd}0td>Snb4NZXm%gVf1b81Gym;o#m`FMC{_Mq{Ve|;S80cGS9+fc?Rjte zvOsbdeJ0D-2R z70ADck27E$^9+`c?0B4)lkbgn@ie%%rxDrsj@YGc#Gz}RVtRw=S>fn3XwO*c7&fC| zlm)V8A$u;5zyTuq_G^+p00Z*gy9x)yk^iG-$F_PCH07e|@i$kh9kk0oz01{GCkwT{@BG zX|<@oDRXzK%cufIs!R%t!c6lpz$$`M%7`4cGB$TTZlJ1e`fNj(*^UY|74h+BT{~aS z+3A(fxkf)U@lneGcV0bPXBsrX0AQ>d06YEOQc>ul%Ib^OD9bNcE^Q&VlyYoRdHlQ1 zq~J{t0c{IL>kKx~lc?Acn9+tx#Ehq5wdX3?@$q=CEQj~AxR+#%y69Yi&dP{+fO%1J zb+1F%XC4U|4tarJs2;@FzWuuO06?>&MIBj$&s_2M3Qa?wp25z)<9&To9z*@wo)6A; z)**K;1l?tf1wU;QkH+Jk?T3n`S6*6SpJSRTf(HOQ7|n3Bj*k-ZdT4IrQodR`i>C4d z?(lfTlD<-hrb7O)ZPo>ja+g?EbM5Rc(pMlo?V(33CTTt1o#Tt;vvTN0BE@@01m8x8 zH9IC_OdevB1yFLF87T9qRv}_{C9k?AcC?H-s#a_xXs1Xi)V%rY!u@*D2xzkR?NG-t z%hbD&tsW16*^wZ<+g+XTvjlxlDO<@{lm%;LhsjAH`t4aOGbV@PnMYESK4n(=F&#)j zJ1XRtU6fr!-)}3b(ohBiNE}5YT=8C^0viv3+KPZS6 z+!2(}+H?3jvUUJCthg-(#WKf?vOot>-(M5RvAK@gtd+;eWq5zvx~e1d8lHkP?n;cUBCISkro0X`~mvD;AYr_)$2%Sd9 z1Fh`?DtXFrLS;aQnPHt3HeNCagq^^+T2iXUW_uN;+@5|V*+hoW5Z%sO`G69L#&;dy z1mHBz*y5gH}Qp5Aui+A-6C zHh6PV#8;nRS}|A!Jy#kr^|zCOv!#F&U^fI`ipdi`Hv|P-vj?OLT)L#5^eSAU#2)@~ z5vvNIEbH|m@udxBQRNEC-Gam4cVeOm`hy_(a(a?aamS11E{U=2LR-3n(WLdc;m~EG ziH=}Ut^1UqEWz1bSorg0FMVpIn&eHt7s`GDl!I%wLX5)&(XMVD2!%!p9#aOPpGw#E9HUaE)XOlRBY6B_NtkN?b_ahWKA%PuzbTSXq9?QY_WVpGqyYL^&l*{#Wt?nyfi zn>gxtu7xdC50D=Z~DV&7v$eAlkqwh#*4J8F(-@8Vx1sOl$<%)qmKoXV&(hOWW0MQo7lLs z9%~+Xk7ZO>8l{jJZiSlG&w{|3d;N=5eQWe~fk_b*aXZ*aqgfS1S>su@VRh|p0h;Bo z2{VV6N!On{tafQ`8Hr!Ni-{N7^R?M1aE~xosBX2CZD`YwJ6O2$n3L`nqcmGzk`jZ` zTqaQ46dkMzVC=8VA?k@2mO5Oa+OXa%+KIJ$wtm$JQ;v!BK#1gD36cM{rwDTRLFQR} zn5K|4V*;usl$G2>P1L8%ly!Bibn4x*6xWyX-FNL#$er4-&&N@O_`Gq*7GE6Q} z`rwb#@1-o!Yx|#t-!VvAnA7}!t|Y?H8s{xYV^V}13E1wVrmtL-+ExgCu3OT-&?wZs z$aZ$YvTiW3f-r=Kn#U4f0HPwQsy9I%0CIBrxa|}rIqA0^e+UzTGmIPL%m5+0be6TX$YZH=bpY?UORCR_2s%6oZ!pDzQjH?d4*m4QFSaTY-dt`z zF?ESf)^4CXd;u0pR+5nXhl0Dyx2w$rfk`N0)(#E&2Siwz$KVYxAjJ)qhzL?E$dM%| z@cxaKxwIGVN2V(!lI}bF6~1O9N9|#T<$OdNiW<-;!LrF4C|9J6j2n4lIRd3hKZ;4? zn+9uc7Ivt=)G%yU{_b^G`l`H70@?^fB!5(>Vowdl)Z)n3sFs^J06tL;|7fmNhco+t zolwxXf>PR}6Ntv<_qXDnOwK;z|YL2%@G7yF>FP81&Xt6y|HF0#S{r1?)vQLehlbdwkEc;#mun@vnKomz-#^5!Q! ze);?bXCg*Lv!ZIYqINphs5jO~AOH}mXdXi2#b{26jzj4uvUzk$x%~dxee{1N(Y$($ zgbI@)Lj`QGn$qf@vjG9`uET0B!Syz=zoXv39# z-QX!Ocw-_bF^JR^!}}5A+(s~E@Zzd=4#6s_WcFGi5^Y=3hj~^^p3VfZJYC_oo!|`k zX)g2Z&JJ64Qo!Ml>p1CND62Mrzs8xZCXpjP@8xPt#J{>~dMyU1^|DkP-j$9Mtd*cM z*DrbLC{jghSjLPCuRNhXwQ+;}E;m)4H_H#K1-cP&9&pk3WvS-sSjAX1nb_|B9PA}J zKVKj2?S?q+%_(r0O`2sWI7~Ck?2%y2)M7X%ZgHFc(OQ8y5~Fr?b^V1tbJ3PWs}1Yt z;Dzlj7!CpglqY(})4ZqFI8Eojo`w#NimfdeFhhbs=N{-QE?<2DfaZt^X0kgXnyr6WOzaD%@YygoEa_b@g}GCDIwR$~z>&&lb3-V5-h zy=%{U+W=AWnbjw{>nOI0FAu-*V7wa@9V__|wWwL?g%X0TRYy*SV1w9gdC$XWxxBrf z4t%46X+;H>LCcE}M!a5#C9$Y6WrC6*dV|*~IcEkG>QZy7;-@n#FCVbb!20{+1sT|h znow2qvfkTSLb~`*ZXgqblY*HffLEwn(8YA5GdQBZyF^uGbn#P0l$jp{qkK=?&D$W! zHX!LxzXp_4Pv!U9ssX5ap|#B95* z%Q-1~beU(nRYCk5)HOsgO=_U1&+F+4sH^42=;s_?n)$w?Ud;^8;KEVjT`G_8L;9QH zbQlLDA7asUCxV655V~_xQ@Tyu;W3CJ_*LN1aSfpgQVFG5e>}7@__w`)eFi!-;f&yC zB&s7bLP3&P_*Ce%74$sOR$TV#p(VJw@pK@rOQ|jp$Ogvx$~8#cZ7M&1)s5`S4cGp< z!IXGLd%m&r@84uu9S#cCWWca$JF~2He3j|g-~=eBIXkiuVQgt9Ou%Nk0j_@JyJpc; z`P1=QqiW!7chvcijfa%HVJ3Z-m2r1mm;qZzCvU0Kl7076@3+idhh)_2oY$=aDVbj zi#)v!WOquYQMOm^rk zXiqp$P8G!FDyNLlP{{yy6|if zPN_xsX|ichLW2tfjQ5LP*SBquxTW5?nLhde!Qs0hQCA2ZZcbfeuTMueD??Z3AjVmB zJjTmo*}61n8eDFnnaWp@5t(_(EollZQVC!7IJNPPxMl*PP<`t`1GRJ-O$6@v8(=ma zWBWD+FiJ*)_9<}DJHlqI+{KhKs)NK)mAs(*!Zd=KZ?uEub!Fnm2a+Fd2`r_ zBQ()`ff*x=Q5W(Q`OL1Opc>f;wOaIg7erd6A`3*c!fV-XR)I0e>jFP?=q2C%x`H5& z>(jByv*&^=aGm=CYaQCpB)`$qd5TQ72Z_ z^+L`pePRH4o#C-ZG}B!u5+lW3{KemMD%H6u{CQ$k_FOaD`A?hodI} zje-P3C?rlM+Nx1EAx&vg?0TS=@~uvMCF0*J#so^C8P65!!btDwc79oK0C@Z8bA;bD zeXe9Ol%+mVaxOWEn!-gRmzqnOS&WgA9ge4nNb6OIq~`0w0qonoE1T46waaVwg(lMP zw5)4EF)|e`T6__AEMBl9T)j&<$O$wC%@&G6rAKl(Fts_aCXWNU!%If=egBJC2JI_J zYC@LGjKG_|U(C^V5Ve29uG)S}$sB4izX|*fa(q~-pXF9o31~QgKIfz}9@-6!My8FD zp*iWoau4^N_ibKnwc(oAv@ZFAUIdDhJVBVZT<33HsW*eH{IZ95IkZdifH-;fwvWfpA-oHvkUFRsJ~ON z=o^S=F%KTG7e(Bhg7`?a7frh+ZrSB_);tG=As|o0g(KziB%E~Zc@Io;jns+|xEf~7 zhU^MM{y~;r77>DFI%U6=`SzwuVGe_aMsaXgmRiBy#2pcilnrJJ;Cx-P4 zE+jQjW-f`8u)TY%#3Po?T80l>=&GH;ntd}58Dzvfi?<-{WM09nf1*rD0)kodMLCVR z^+dwGHn>@T$_I^*w;Hruj(5?g89;)h)kh8rE#4O9GldiPo1jvY0Cv*vaC%3lNl z8syMD>v*Vcrj=BOvn;@J%5c|;4}SDi`+_f>BEYBx9mB{vRL*3EGAgsB$|ylF_~W zPzifG^Xjj~;X{I!?<~w#c+u*1*D-Fh(MNzLI8sP9dD8>Y6Cm|Hd~gzx`g)>G3Yat#kSL%Dg31H@V}p z`Yel$W62yVSPm?yw?}{>sR)<^jiy_Va^mlj2JBa@)X~vcvZqU>xWF4ONc_v53VFs7mmmn+T1jgN?`_H$mMHjDj;1?MCdn0t3{I(MUkMJi z;u7mbw-O|BEF&atzHKgcW#t%zd5cSrH^JXtFURYjc*9dN8r9>J%ii_SIWj}Hjk+9J z*9^Z~7HrII^CFcU1T*{(XOt62yr%W^j(Z-oCaZOen+n@`-m_(w_%ngJ!jVS?_sA#@ z8_bb+pD{d*o_p;EFZdTV6zc=i3y#3BO^Ip|7s6jwB(|4HgN>Hl1|4=-)nnibd&0or zCe>_BCgk;eXhD0+xM74sH3%&%k^L9aU|v-S|1Y9Jpr2v^ zbgUlm|6m%#T@skWH7^9&2QWFW45D~b3K0?uIpibUab+WD!=3JE5O#C26#AV)&SyDrF(WDa z8CF%T3W2%fgvnn+t>vy{-Gqe_;+Pbu=NR? zvPiHorX<9^m3Fc9o6C_Zn)wYkpA{tA;uah}8+K$lA~ot^Z--xv6AO?dNU5-xU@zf< zl0{Z5E#)lkS<}AgTE`*mg)D*SH*vb87nE~Y{f*s#v=>Nfq;1kySyt(_U~tzllDHqk zwP3<+pTZx+&(oZQ{Y@_1l&|a|(3w6Q2n#X&Z+3z2A}A2Wam*^Pf$+Od$w}dRzDbQT zNQ?4H3Wj|(oRCYmanDezO@pg2m=vl3mQTQ?*h(n2q4Yn<1(!P|2q`dD?M8lu8ZjtQ zy}aD~?lVlb!Z}Zlb>Cq2aNH`wPK)7VnIQ9Xdm?$Pz?!>54a&I98i+dy29akdNi-GQADgKuC4Du!Aa1x zgCSig;Mr!(Du+*792MpJ3QYx3SZ{wd^++tFj!&*GCiaodP`L{UHdI>w0}Nk1naN4-G6L`_NWr&907Y_=XlzeaGV@SW7zhkZ^? z%;RDnFc+0m9mKEkD*DGegh?xO6s(#s!*oqFxnLT!;MCpi8Cl!UDzR>1|XBC%8iDf z@kM&?u}uw5&(cJm9J9XiXNMM@WSVR8v3TMo+8%+94g~2p2~l=$v*6@vad_#g641Q8 z9Ug=%@0i;wf{92zUV4+$kjaUmrIhodOJXUUd8yTK^3z;JzEx0c;3n&8m*W1f;(k(7 zP_Be?AK!Xsp+(kb5GL&0@8^o^6RzYtT$xB!L#fhipsZt_zq%zZp z@Do7Vfh7bL1^X-InmZN-FfpNA0?qva#&nYfRFpXOp39Kq8uKRePn|L&XJZ<8R{vD= zch*EbDta%exP!Fv}VNi(HrjvWzZn(}uKev>^!AsR-H!WuWAJRT!C#(f+@7HFZOjvW4Aj*+* z2E+c)K^+M9nh91S+_C0ls$%K*s4l8)A+CL1$Rl1y7p`$uOcHUCY!VauNzr7Yu}q@) zp(oc87}wbrlC2j{?CPzi6Se51bWSGI+7PC>9xgjw-dq(XSMy-{UIgrI#>7oT^o_-R zL1uBFX>2zTfsQ@oZ+!Px-+^I{5uF#>(g8=af!~YxTN|Lu&k@xyB{>#u!^|DM#*w4* z0mXb0t~(xbd(mRf)l*koB@Ys1YSW21TPpOku1cTt&!}V_|pSeV=*~>^{ zYLtzRmjhT;n(5V!Da(=+oE~?t%G@#hd;D4nmsxL1VC+qx$La}*?>@t>Fbj&uU$}U(o%dju}`fJu%r6p-cQ_zk-_$zzjeXG&Yy+f+im+T@ zNGjBWRNwagPY2+>N-L<&W^Q5!tWV3T?UcFcaDL zz~`@pbk%>G-(&!^Ckv51!ngfTEP%XAt|rSp2KG?NCR&Q=ZY;IZooznd`{Db@`{Ufs zRcE8F3Zim=>r6C-a4tN=V()EDBZ zL|f4iMuO$7uXVHQSLVsNEO%rr*5rYD-iLfCO8C9~&>zQp`&Kx@9~&5Lq(&2|OUg|R z%L1^WUD!I!HowEP`;H~h%miKS;>-YS+Ol-4x(TmKmeO@+&e}d0V8Ri0Ha4!fS)YN{zUQKhv;EvFQeuF!z7th*eW zM_jZtT9xgIf&pzoX|^<`h?nopg-DUE;fAKXbbO&=znhB!!92sorGzPctfe50Oot{@h<(z652d>pLt|NxST7o>>$<@f}{P_Ne zI-92E7X1CrubQS>i$0d)b7aK-+q zj0Vd_m?a~zypH|yE^U9{VM!RNF0{oR6-@^RIz1sCwD z-un;S%0|A4yrl?B7{iIGs})VDY(YrJO4+K)f*cfwM-0JA@k@^;Ir07=kAi9?}MIrYbyz(*Uwa6qnHoj~BD3nh^keC&)fAhJ9HWCSJ(UxOsne~T8h**u5PqF5#sn#;6sYO+PfN_~Cs601@T$d%mbXoHe}t(#4-lq1@}b@){^q<~<@ zl})ioy5=?MX(%xYYe%ArgBCqRsmGo<)W1vP%io`2d}wLLg{?Y~PeQpCD%I0C;Xq%K zBamBnV-AS&5mlm@NEVjyqM6w9xER-(uz35X3a&%2HvdckM-9gC1?3ir&OIlkl90|l zQ>04Bq6s+LdOwV2X_s0^p^@=4;i_)^hPt!X*qI-r8QZ$;dW)fNU{W9LwDw3OZMkRM zZK7n31KM@NH8n7ENah$cenTZ#8E2gcK~H^CDf6Um0d5G#TYy0+8Y{Ilyf#cz3}4`X zt;cY8{jL-NAGAX#vVOeRa%`Xh0+SF+ryU#wZ6}>u#U2)(Rc}->G47Md7lHVsJA){K z_8~D2{ImB5#LL&3)?xdV{kzqieBqSFL;oT^Y!2b}fCh2RV?2-ia1kzpi~kTtwTVR2 zi6W2wMCne6pt0qB-Wr8xwd2;Xvb(8DK!<7@wdG$Haol2}xcH=4iu8c<>f-d0(h^R< zRg|LX^yl5@RpCd2*@ie&`||#5FG`kOdx~K)>m-A=5LZX~l6d`*l`rDSpEKn&vi7aG9VQQ;62xCV|BQ0APREY8UM-Cd! z&u~!VfVN1iVDAsHgMpVBBxjo%C5!X`xs2SRcV>Y$%U8}wi*u*xa6y9)Ajp2OtcZ39 zY;yyXZrI52=UPL&@00i7VG!`AtF$B*ZJ?W@AW+cNsrF$j?1AJGXLjf5aw|Q|P85{% zJ!O;PwBtiW{?~O9U+!~2e!2{n8${UV9IqyVZF-6{qjivS$5>Z>R*Z`kpKpUPeqi;i z;--7-G4yp-p`B<{f+OVk;}SpXjO%AK@4GdC7$YLnVsF$oP?I+6tiY1Cqx8k962szf z&0!S|A-`P?GG_!Zhrn`2kg5{$^VzxbC}a>9l$%V?Ix2QpNyy#okFuCjj~3H8x<0eg zOp;wD2nUQDoysoYW0FuUWKF9T0hmLWfKJ_HVi{+_6^bl#bUEN;UUwAi@lxamLLq5< z(+08>a+Xb>l!SAV8X1@ewb=ZJ4z=&}(i&36oC8Fc-iaI;Tn0m&go$rjo2`)#xEhy5 zLw>lIb+GLJCC>Z3AE3@{TmE0V{Y+`X=dJI5H2>b&Z*xlh`(jU`VGncK`oC3us^YI- z-#h(l<$;iNJH-Ms`B@+44UWdJX|vWTPQ?xW(pxm1bW^mia~!>~Y?=&!unO4l=Y9k> zAz27pnv4*pfLsl~U#kTeuZL{rmO=#4B-6u@ao!Lo*?Hupa8TB(EHe?3GElK^O*uTk z*{m6%MnGMUISoaz4);T$g%-3xYSyH2W_vnV#n$Ebdk1=VjGd#CbV~^?zkr1j4Aof6 zI4Z*ig%8aL5NOCKEr;QbhJi&Cq)e}-hGjV6o!}1Ji5Lr%#WuQCJiWRL&#e-5k_m2{ z{|Vkm$bh;Ta9K}~h9YJqYcN2WlMRM!c2>iG#E0;3E@yV`IP+JF8x(yD9u2KKOn%)q ztG|_xi2*v{!I)0UnJhTb2>X=zQU!}rn*mpI!KVzY8%fp{2+Dy?nP(cAOyx{d<^d&O)J#>he`Y8JQ9%Dc}nfjaD zFxkjlpzi$-hf|78mx1oqV(TQWo@1(?HEA)l(oCgkm9uLUbi+vZ8i) zG;VOPcSTqs)RjC1!AukF5?H%54G{L~kqD3cU7rm8q!ZnINc{@sA%&UQr7?%r-4KL? zKHfwVS-a_OW<<3x2p@oJ{IpAw4<6Ib`;m@GmfhhJm+0LGf*#N_Q$%5*yHKdKrha)7 z5_~3z#_%GmbPu8^>U;G|q62QgFr$1b7oL{G$W|^WhOh7n!+913PY0*%b4b5%rgTlT zZ-rC~;r92OuDOF00cWGk39Ooh_AD;m!pi@+GZ!8fsG{npj0osP()7 zeD`o5wq_fvwF{OTG@v{Sxs0ny`%*S>j(6g0o7c`|{}&DuU*FD`*5#eQ=`5)9-y#HA z)5aX_JVqqB^0}$*(2^bYo*LR<#_ag!$Hz8E7M^;NX(g7?wNBS17A~r}nA@pxw>K+p znc1a?n(ZoDa3KxSpbf7*5>94$e9NTBoq*CkiOUj;%#9Y7g#14JiGl!7U!z1`nc9Zc zQ~RO)`dndvkTDI^53eC$uqzFLewDd`vhto#kq=0xX^%7Y3lP*B`=RI{6o9S41 zc{pB{I+lE$aC4PS5=I(UYF@V6@f6gm6a`$l4t6XIfekB25hIQ5v*=JwNhUGk0zC6p&#gsuzy8i@L& zRKnoFk_vM>?hZKMkXo29Ee*+!w^%IIxqL-Igq$x-C&T0+X@%p;&iw9S3D1ytpM*{d zPK*##K-$Ag0}3LtM##@2$=)U?HX_ew-R(__uPki4c~wacU4im2-9rzO=Sd%Vv+36?xI%3sh>@j*{}_VBkZ8QQ$bJ2e~_ z&ku#*)p;-3=<_Ahamuj9t>|mW4+gwpoji%=YgY008s(frA8Sj@V?iqSwJ{=!uTB&k zsf=zot3S3(1=trC|Qb>Rv{Dhr}_*FIYugsS=Lbi&hUZthfOn3^yR;FPh z{;F6}^Ji}RUB#nnG4pC^qu^~Xb9;;tg6joZ%X}C9BjhY_Z)?E5=So;m6bO|Gw0p+c z5H-ol!^;5&Zs9129FU?EYRnxJOq?jWBtHs(8+VlQq`L+z%R)*W@>{dieL_T)*grMfu8aHiQ^5;dTY}Uhx0Pe41X(Ui>hNd7B z*GO%NMTgR4*lDYoZ$3pD2|iEdB9m%X$tDFIu=(BcYs=)YokaGoy|{N|2%achD7+22 zVJsmJ-us@Gkcl;wm_Z$urGc%Yy2TcoE4>dZJf8y}A(FsKYUGRfZx%QLP{lYDiv(Sb z{*2?9#~&YgWLuNx?z>DlZz6gAtLz?W!8x(?NJ~|+rGmNZWc0tw>s#h8h*RNpJ;eoF z?ZL?&*GdS&D+K>57B=b0u7fBceXw3!i}f%Wvc0|OSO0xS;#|{?-5v$iXMJ!-z|E7k zBudJPOzc4`<rp2a;J3Ho&Zd&SB9g3}f zsZ~i<&es)B*?D8({NluON$6KNpK-n4(*gih@U&iDWOi6%*(!H$BD;G(Datj?`7b!% z2~}(@O8q}y%0v`q72uS5)tFYgM?;V2h(i4W9F|5$ZRT#zAc7p|n$r41IIXC)!}%Y% zY<^KHO4XNU<3R3|2D%9|?7+pK>vIq?k(;4Few0b3FRdP}#{~1)tzqj^OaF-u!KkT%$mwio$vy!lSfBKEG zN*F1y)e1u*ao5^iI-lc@fOiAFs&%wO7WVk}0;}jVB$*Q|JO>{dp>++#(Frp>1k|$+ zI~sxn`+>mTqFo2I3v9_DyC}F7jbdjQWeeaUEtU~Q+L<(cG%UI%s$94lP3KrB%L(0X z0`8F9^HTGJy(T51$;r?E&+GYr`Tzc}ms|M{`H$^CHvdTf@%WGZtDifBUv7DpaT{S` z|GVXRrNH-Jm6D`T4T-S_N%oAEvg5e)@_Q*N#coTs;-ACwq(`Bj)fdzi|J<6YlC8Of z99j^LzM**v_n}oEP7&PdCIf^ySH8b77GQJp6*-|qR?B%oN0@>2>7=zYSP(LlG+8H0v*8)yI;@} zV9a8@l#RfWGeWhFi2IV}CV84AQL>EMLNi)Rt&I)+10O~*c`iIX!`0uj*iIO?{)58` zx}gr|Ob!7EBfQXdMk2fICeYP&k~qa@5~ox1*M=8x!UTG4N|Bk_CO_yqogMLq}2xTPK->K2VlJC&kr&6npx0h&uj%;!<2!#fboQ)Dz@sm)zT z^;}TJ);u9wR<1++EVP{RFWq1gpb^EZ{S^ZPP7?$+z5$M z+@S|wJh1(XS!?I{;wWF-{0sQ)#iDZv_TQvJ2jfwCZI#7oH=Wb8 zHmIQk`?eHYrZ%jSsOngha;KwkA;~mb>D>&meu*eko_9i+CY@!{Stgxj(pe^*r7<^? z&NAsNziypnpe}ven{-gWU?Z|r5=c%0*g;E+P^C(~1S?pdgiXYD+swh>s`8kvpL!}} z_mo%|q2#Rm1w~tBB)lBqRM$r4j^>%;bC&K@y9jwB{$wI>v<5g}ncHm;NAGq1?+@jC@8 zO=TbbEq~b5$4_S_?z()Omyh%Le6Gi73t;vX^2~)Ym>uuo=4KlZtg;pk*$dX7eM0rw zu#gy8ceYbUsb7(aTQlGK^wW%Av)GDBUq!48*O(n&UAC>K^e^;N3d`Ca^b#IfQubf% zkQuV_1PjI8p2>+fWez@s4(ntC1Ibx&!Gaa$CZA4{yv8ay&hNM^||{TG)y$F zRG{w;%eM<>ScWT@JS>`x8Trf+W-15u_Pi~6W^gexrjAZw+cUk-`IeS@t!(=xP)lIu z!gkSWDi{(87KnX#Vgt6Q;p*i>DLg~QAi0-`>Nfe5Fk zUbe1uH`~^kj8a{nt+aHUIvsW08QLcCg`}UOg!Ji)M;})4glt*Lt8J8wJo^z$`}=|+W&_s`3Ap@TGdOOkX;RX@i4}_fKauY!n*y?5z2ot3B~cKmjbo@1kP$ra|^Q; zIHzD`fS3&et889ueAjY}>%gNoF9E4Nw%b-kk|k4*jG+j-v-8(DbF?xX)BpiBR&4Fkoy7}b#%6c6R~6Pj@cv7BJ5kY zKU}&4z$ARtF!)ZKI3Z8}ocCo^ywPV=2}(qR-@1Me~%FsaXbn z^w%kFo_VzPL*bzYl3U7HL(Uo4Zkp03WfBhWWJ61uC{{x4Jv@~B8ydk!hLvm)qe7&F zR%#>4qcxm5Bm3N*F7==AFNr=2yj&5JejDq`y%%Y?B;bMYu3xGFhs_Bz)#|qTi2&6W zddmpu4c(28HV#Y~c4ZD-nZF5@6pW}e9$DEhrVzB|al8tR%S?@zaaX)TPoTM1<+P9+ zp{^X^;84Jh za3O_+=9bbyN19|IU>*$92pJ3ypXBV(vYs9)5VB_Nl3`9Ud;Z(!2I_y?7~n>oxYi(R z=u`h_g4A0hZ6Y4cp=EKBR=qYwZ-O-I@tRe_QcP?I*==r6!CzIHMP}rY>HBp)R65@d zZL7C%qth0YYPz>%8@l7dmhcXeY)+v*{t?3(O2QmlhMyMP3yf^GwoZL}gYZ?u=?LDn z8M6iz9NU&mD@HD7Hu_PTwA-1J2WN@25GJ6M9n}3y^pafM-e8XxKi8c|$Ln?yM8itFC9s~UrME;2$g`4b5> zTYNN4Ybs)y`?zF7gWvBY%=x;Bhq_ZUz{F%AAyKn^;T4KXS~U3izNAKj35*7*h7uh0 zCNmnOc=BXN&Gr=&A2nMfM@Bs+Jpts+)*I%G$?C+8mCfQ-iGZqTp<>F|mOSqZ8*jE) zxfl~p?j6pU(DFh4Xoa4S3^Q-Gh0`X?CHYrn{6N@zu6+CF1C>z7H0vpE#`KH%^%L+ zekvk!L$jL^+HU-a2EzZF?aQjh@hJiZyI@lYj7}FYJh)~$fvD60gCxDF1p;XVB5rU| zX$Inzpp$C3rbdo0m3*`LvwYt*sMNR)8#$0s9st0Q<55LDb_GasANE~DZ3V=+lE{FLsVsyivRj9Nd?Lt;n$6{yvNw6)- zEnbn4s<-P0#i=dF85a6v%c1_7ysR`)70r%vverz)02s8P{ZN#DF=uN+3z~2hyeN*> zMj|_SkjB2XAVKDNMz%EB@I1y40n$tYgicSEu@8Z5am{zGaEb2Ypy3>wW5YAqnK&&!xiSb2 zNnW(XTVF1z!-ggC3{slK$>JyjFi7i@E^JrsCMQyM_QM@612%b?TUyH6=1u_SfQU zX9HUu;D$&c(lgn7KeEPDCrGmpDT>H@M+>gmO3`Fv0CV^YV7>a5QD0<~t;DI4jgg3| z)SRK(tsq4rOrc>mW+H5ccvosjNF2iq5ccl`#y;4 z&n&5Y0dWJ|?e(~|(O9*I4ae2;u+^+MkThBeGl!npdSLrHWwG(3v`G4 zS7S&K+a=GjV=fM^{-!mOdXZB4A?VUE6B~B)wX1=T$ZYMbyFQ85=fZ1v`gG7;HW1Fu;$#$$HFH->sR<`4b>Qtt(?f+(_04B@X znkv3?eh7PQR~FVR$!qcwh!MWsvXoQBvb`@IhOsoGRk{D;PtSZ5NORMZ;-)1GQRzF7 zgzh->;t0In+*Tq&cYCpd(6Wn%>VIfnqpn;T?vW#Q(V$M#)$CMB6P~Xl3h>@HpLBJ2 z^=#n}gxf-H4eNi`P{cm#O=;3D8@B*3ehn;#kkfomsv;3vv^-@@Yl;wxx-1!e=o|eb z?MR-fLhe){_n)CcuCd<-_umf&_h)MOwF_bEquANc9Yj#%y+@v?R8gsB$T5!H7&PpM z>R1Ms@UAFWp7u7E&@I-N8|;l8s!Pr22~BH`;WR5x&BPIMaJY?@`8O^ouQIyq0-?p&ty%?!F`^BJk;I07`OBro2)Sdcru#%YE_s1$ik-=#9R-W{mEb4g17gmY{>bTKHEsR5iVBLw6B^j*s+Xp2ZgR zIp$O_ZoK*`6ZC}i zt9JGRH%zbHoc>akOsgI-Mf&WshuVw&JGr8Yq7q{Q*L*YY&O*H}mJnBK(!2RJGZLjT zBp~(IVMtz^lm1FFti~2P z(OSCPXpQYDa~KZzKFBHfxxkX`#$0%}p<3XOZIvnTBf-Mzl@1->1uI?ZHfoYKk}HO8 zo%F4Rc{^Io7eac5CG{S{|I)W8wFpK7rVTE93HWKA+n+*2ca&#z$uq9@`gDG^Xa!G7 zn1Cm3_mS#D(;ol_HF>EioIh&8^J?KVhj2DHlY{~lkEX(56J8%`6+L2H`MbARtXiFm3OXmB4y-eGk}9AG6;QR%})=KD-$E790@CP~Iu%nRNd-d|ghb0x~yy1o$i53^0)n>R7~ArV=| z5^JM{fmzZ4{_D9;FmoTbiQa7XTb$_gPuCY$@2?lz^fw#vI9Fm`NjB%oe;$3A^0*KZ zxQU&_kJhkHzL#h*2Gfk@0+y0e2sI-mHKu&8*dXT0u&S~QXIzzx&6@U*K+yl`&*{dn zV#RRC*v3jWS`3HNtdeJDwRlgqdX691o(v%IR-%N=_ig@QDuLlb zirIxZOTNIaj5$l7GoRS%*opeYRv0kWj(&o4i7Z!uC=QzO%yjh5oQZn=0#g+SxF1dr z_3e34>vl4KYTZVN^&ng{RJtrE*Gs!eOPUIal_2z7%;Id4G~M$2o7jp&3Rdu(A5x!q zp8DpuJ-3(eWLtm2x9hBl#`j0<{lk0D6}j~dwmpMU8+yB!f6wOl?Rk;bGyZiPdSA+@ zbKs2o3Z|VXeR1UKkVzFf=`6-*KS!PEkaJtw%rJ;;!Q|cM)*DuTT+KUE*$=Zyj|^4nAEK|hpJb`ja#E;kjcM6sq>uPEksqlr=EngZOesy z_Tcb!e+Ju=0fQEzs4_pQ8nEx!@gomQqdbjqWz|AtD^f<5+?4^>ih#{$3}tZyG}sb;r?In&iUash2TcA~>pq$^2nMotjRsj~+Ry0v-M zj8jCY8;V}Uj%00T21f^CsroW0O&Y_BHlLzxw{g7Z!>f%!WCB7gbfG%_Qo1 zc7A>a+k335Lmzf1is>E%z7*A z1Uwod_$mQiv{RSF5d#D=ykcpz!WH6VxxO1fwkM?qb2&Ia2kg4N!y+j%ewd6N8R^C+ zkCfemBPkSdut32bFHkUt43zl20F<41?RExO;&=Yk)megT6h=QR4xkP-+zqrQbRk96 zwIJGmX|^5IOwyII*l~Ktabkk4NN@HyB*XZwWq$am5cE-a!1Yt%LT?D9K{OORMMuq3XtFl&-0ij;AQvK~fZ48`QywfikYujgqyj zS@D|nV)t)f@hX*5>6lMFx7IR+UrPLg8<&S#`b_)gWzIfEf5S5 z`gqkm)_?%@3fOJrE3nx#(2i0-*0f-HMY)YT>4wW;)_Pj-Z5g}0C=Km)J-{7Lo3 zP22YrahyH%wi7kUjVdp2;-WkuSbLRj)B#t)D z`nL3&tlp3fZK9;RwqzaTCEsv~W8ItZd1$CU0K0rovn8dm z$G6stL7QpdVV(FGWIi5@N}^j*@@C)o{S_;Dx59cyeh+!AEGPhoaEO|;_!L82)k1BT z0UKJ7n)J;#1L#z(f$IA-QL?kR$IXa=`UH-b+PL&hG|&t4;7cMJ@?M_}nGv+d8z6$r zC*%LeH1jhWwMw+K@@(Uz5(TZ?i1kjD#8|oMZ*L>jh^hL(`K+BCm4w?ZUT8K4B^A!x zh+I2Lw*fnPrx`OesEL>bvZ;a%XC0 zloxTQZ)x12FCCO&Y(@>{e9f!=q_@a)u7kVU+~sG?)A4EAN!HBnJ6;}jf+lv`y8^M# z{&d;jV1MF^hLOu9t!q{VN9F)5amt|_LMSE>6pw)P@O5NCJtwryR{{@Mb!cEf?=ku5 zlC-qBXRUgj)ZJ>qBCF~7pzNMMvFor5Ea>FlfB%3+d>9*}2hmjJ11}*&N0Jvx;LGO+ zVR(4pQ-J!3h8f40ZWz%mGwDKHok%aV;3*7vRo-&#jMC33ohamYP{x?+cD;3~Ic}VM z5t$kVf6*BijcAud@>N&0v}U4{)iyE}o~FZ7XI8~N!_>u`UGCxj8K!}id0_{rcZf~h zHc5zOlA1;BwaW_zeit3>#tO2a>fBma-94Z1T0dEAn~&R3nmR(EOocG~fq~?1N2AI} z$=0kcHB`8tX|c7Z8-Gv_8!z~C26zN}+}qD>L)EslQX7!<(DOS&>)1we{7{{*{s?_E zV3NlQ8iYMAIoWM!-((1%{rxb{DvH9559JF3_M#`EdP7?1H#(%ih*y)p5mPf<8n8t}L)_Jqm<9;V zgtk;>feH>0btP*d9#8|6tyoJnoapRtscT2mW-CLfGq(MnF{lu=YFo}$$NAm^U7t9k$A|a~Je!hI$f2qF!e!hJB zA3?NnUoa1OUUK>A#Io@RVA9s9 zCr*ZeDg%!W{Dg*_UUD5r093DzvvqRV!Ap2fuHN2!e0zCu`}XQbV&(HtF+C@@!n+kG ztpd-drsE-fy~D>`y9^B&#Y=xIoui> z`09R6t=X8;1WUO;tIi4e#ZsKUQrgV`94C@!V{7Vfx<};Ob6a+cu%hu$A1!6W>claO z+5~MykYo7K*ZkLy?+*KESXam&z7(q!Z^@F%rAzDc(Fkr;?_ZHXeec?lZZ)K%`!lD~mvX=5H zrttY5Y{gns;yw@0d88Bm1d-JvxwLyoJWVKYi{*TsdH0dril$s?LyP3NxMX2dJ6Zp zw{^p2>@%0zSD|wkMB`8IsT;R|LBqPBtiuZ4t+vC6C9nBR2(|7&VEC=7EdYw@Y-Df{ z(0g5J(HM98y5SpMv1yv6cFF7efIL`J9W#@L#dg}D)~{@X2)PNddKG;5<(AY;30rb` zu<@X(5s5XP&1Tx4^5S1gFD~c0DON>t7OobGWw-o4xB=*RfO9f*I(bEE2KWI@48dR~4^6X@YpOnW;HpLKy2$ zcD6Q*RB z&oAfK10pZ@tmoFZ`{6`O_jBvVVS%RH`cns@Zt~%pWwE5|i>_T>aao8BLmM>~Hq5xz z3AddHQpn~1Uff_gL@?n5ww`k)Gc2I-S{fNgujA=5okwSVGa*M_`k7xk8#~WKmVhn74+1HL;X+vUPe~Nj<)UH z56%P7Nv2Ohc%p4+EmfCefuGLIrPmpG(H83@>eG=z;-Rvphu^tS_~~Ci;ti3Wk61@c zuc9~lRq~F@79ub~mL|&5fzh#>yqa9!XjsnmKWSMuKf-xt!AKfyy$MH#hhR^7(Qeng z(5Bk$<&Bmh$zOf|^o0NU^AF!f2J@*R)ieDaPk7&SkfM+q8>tnQx#w9kYZYdd8ZNhI zW7U~ceBZO;_QBA>ez5;e0-MOqU=txxD$IR0kzlKcDNMPW$idbk|Mz%HXX+%G_$KW6 zeJsuCCDC6Buzx6@wXGqp)wibAsnC?F++Pzlm;&>+8E6XbHpD!saygpze>pM;zG0ivDY4?B=0dbsTbi0$px_d6Y|I{Sa zqd`JoLP<(=I?VNzQRZyQi!_si_}LN6gGhU$G5Cicr{#XEmuf&CM1xOCzv;a&h)^)! z4Rgk(RQxYxmCW^ZYtWRj+Z4NYB11g3s8CBMB6DO>!O}@Dnb?VUyv3AxNMRG4Qvgc2 z3C;&;@|jG#znRC4!LP>0Zy(exp;Rc=v(n)@21~@A0Qls0qS3Hb9MYFPB&SJHer3of zR&+fHHmSCR5ZZjt<)4qx|2_rFAA;|rwtxsM|0vXcMCcRf)0D4aUtY_XXx;}*g9W2nr`Q6nM5PcL(okQ(r9}SOveb3BxFNQa~`84j`Jdi zQ6&TRW0cf5Lo!Go;Xy{pZc|*zBaFx>!5G$L$Bm4Vn8lxrZhRJ#GHzN0Y|BYbWt7AT z-sLHFWt2<|mnq3vqS>s>D47Js-Rw)w?gItE=j_XY%+9`K0Z~K25Is!PG*0<|mPrC~ zh&cH}hR6w;hzS~q48ak?h>#E>OX6{;BYZ)VNMy*&BTpKZ9AXJy@DvliV8|wX!E;aN zOlQYZonWEPkU7d%?PNG|W7FIWb` z%$C~#iG8^Uec7u!@)S6d#Z)+~8d^sqC!_Xc)FR6tz^F}eDQ4QBJy5hMR>cgNyZnoj z{E9hd?IaUppJ_2uF-USUA~_cWsrlP(E)SzG`HYM?OG)WVcy7iF$$b{bG)rTSezYf_ zqc8jEW!W5$zTns%Gxdm&8*+c%M_=|-t}?hFgO`x3keOX#)gGE*jwP12V=yD^7h z(wF0MO&-{YeKboX6XmdG8H#1D!}9svkZ8dWxN`(Ik>NWqxBp*`2j~fJ_f1A;0%Osa zoU47j_u+K4|2S9sbHmkfy4N3V&FNm>m(#ue$pvz9f$Wyky?(D*r+fX=z5eN5KUR)< zuW!_IxYS2azAxlX->_s|6ou&O_Evmi$*Vn-(Y7Ns*}%uM@Mu?ZWl3d^_Jw)T8ae8O zsY_NmiCCAfdbw4-a852r!wSCU3~2Bvd=AsW(t98keaWgjLyS-d#9D!EVzz!;U6fq2 zspx8fIv^}czhxY2*uRX%fonaq{5Ggb==6fF#qfgdRFbZ_K`c1-4#7UJd84QGoE4&u z@gV!sn>=ra66`E8{$->YTXv1(J>JlYm%QD&t8+tfM`4|rjz)>Skbaa2U9THKizSmm zyFmvwnUyG+Htz+i83!3pJ<; z^Z>sL|09X0N(*lEmv3>VA8QJ;rnPy)SK7l_DIHNA4gDX$O>$p}KYve_qHE-Nm;?Ep z{tWDM>@As6EL&1BDqHg1w`9fZPN@cncLUY@hwstL4Xc!6)$!fH2eYcy0D5k;ag(FwD;>ddqR=Y)MtH^U7^;k-YO{I{U3y#rm6BYP(Cl=cllT-c= zc(R{NZa*9ckD6$3kITbjM~yz6);{i)HI2+0)Rwok2CbX6C>U@KUwO8;T7cZdkR3DU zI*QO_``P9X#x~#M^FmgDxqzHC-7DQ%5?+%7Ns`%yL!0 z%Lpmc7KkCi1diyRPNG4+ew9iWi0=^+?o=DTDVR~4B27|1en{mv5!{cR>Kn`t9?X30 zymZ%G8u5=-YFmieKQ^&`Few(>LkYXm996J1Ccci~F(l5qW`V&;IQ#hKs=q6Jq@|GL zO(lv?}}{68M5>#V58^-(uVGcX+&KF112mJm}BWiU0Ac(=6J|Kc~4vRK+!DY zwkdr~aX0iAH`nB6ka!r#Y9m_E$gaeSax{}t01h2i>&A&AAWxk_2|ELD$2w(a84Kuy zba`Z@!s`@)6Uc`8dJ-EmiU~)Szo;gX)spL5pddKG@P`8VEJW&p;t^YU@5DW??Yb4_ zO7o7!bdi&$@T{7Gl-1jsE~?mdT7ToO=)IbBOuzNI1r-vrU9Hn{Cf! z^T4t_Y(2rglYvBfU6-o5O!nfFj2r-2bs^?Dk|Ta!bGzilr<=fEE*kY&9emU~j)vbI zvSFG^-{8&G+hE6^n6uux99!e$VsE4xA;%mcgkMH$qK6 z98`6fC>cx*w>@qKtNVdR-}&v02Qa;m8rhu)m)(Q zg}Hh$4_7*iZ#=&(A~TY^7thO!$sEbpLuTV8Y$qS+aI74>tjxPGt1eN~c)s04E?q?C zc$Qr{gDxVnms^*}w8%pfPv^*?vfr?g*oaKG1M}O^L3fy3ww%Mo=cOftl6cUg$VEC($qGJcHgmwATHHw%u8q)>F0Sx#;lK6n)qxQfpn3v$Ae4UgxHjc0ztPz@{{6lU14az3D)9jI_2*!syLHPT zMNsFWaAoKlZ*tfKy(hKGN{TA}+B9g))#9AB;bc>0bPWON>Tes_lxVoUI9jkJ-Eh%0 zAh#9d8}pSuw6>yi&cZGcFzb3zj*Qd?|D`tON@4vdGxnKB(+PL_%np)8?R+%FYa`d{ zR76vb{uVypSHL!N{ngh-IC@gS#)`Wxta!0SlWqEILiHMXLnSN8ds@?buzg?5Z{ECr zsfIA&-nhORyJfWULHviq@>2$F%s2***2)7jZ0vy*M?2q5@EwvU4;JobRX!p)vafRf zb~P<~HV(TNkv8sa7=1yuwC#GkO<&=Buhw|OAX|N*@e@k(l0Eoum$Vl0V^^_;6jdi% z)_`52GdBBk;Db? zny&e*5cNh)H&M&;Rrx2iMa^hk&Y+V*ZFg#SryI&ErDJ|%Y7?Y8n}6RWC|X*Px?3$+ zqi@gc+hi?D3FaZ>WV+n_6|dEJYz?-=E=X}pn|l^=y@q32vUMf4tC3rmn3HtcDY~Xg zJ&U&Sqd$ACgG(t%6bMpnl6F^@tf@4V!%FW^yXUbnYP1rxVCu;=YgDzBey9!LQXS3A ziFS8b?y=E!joH7=KBHHQde*XL#cO>U=pl@a#!d_U=#UrflJUk+g^(3(3*&gzz`3G% zd7Sa<>cb{%@cu#VJyL7drRw>;T5`<6(+Ag*ZoofNGN}#&OQ)f&qGPmo?39`9Za}?n zVZHV~YQ~-$nHT-<`#l{$nT8de=`Q&1(dSJk{YW<7`TyyWoEcDc zVQyr3R8em|NM&qo0PH+#bK5wQ^O?V*PtKL<@W_X1JTH%i5mV)F!sd^3K+llkz}h zOTq*MH~=VHoALejD?CV1FUxkaPEu=p@PlkLx*Poj0Z77=5z6q2@>E=@5U+C~HOkiE zh)5kxNXCBn+}3Wl+xxq__IJD8p8wrG*x&h~v)kU;-rwKb+1~%5-P!N#@BaYp&mQ>> ztx%fCAKITi);x3HNh6#KO*9o;#Q{J2Ncs7{L|%doMfnSl3)xDdzcAMwUCq#Lp4zvv*jHnG9(H^gP2OnL6O6h2FOrLnkX1Zk--*84bo$?mAJs-412a-U(*rf2IA82`f;25eKlw{e+n!$8QDDi#GBa6Y06r~(52eEsfiI1n-;T4M?W%FqBDVZw++7!yfJpP>RRppTFf zr7#7`wSb8zBxuZXMl`A@2sYqQ@Btkb(rIomWTH=)cS)L|`3u1L08jB54yn>|5<{a= z$AHj7Ii?A|bA}6gM09Dzscy^03bI6r!g2`7X%vyXO;_&Tq{tCB>C-5AlTRl z5=ISpyoe6KNHK;V4}RPa0BA;ro-K*FP=<<0W!^0>`^!@HVNQ~dWQewdCc_xU#%Fa5 z-VNyGn9#ul^a$Zi7f6~)R0|k^2+;#iO8~oQ$1N=BLE= z(Tr*nprn7I{LmKc)H;2us|u}Lyx0V2B8RB0VN^O$PJ5|LV8MInh&PB~15Sj-7(V^E zkD5gP^Reio#eIWuEu0fj=i_#|?J$k2IwO_f$RexlzPm4t4=81yiaCjJeq{UG>B0For4 z;MyQcq3;yVt9+@`roZoQC2SO5it9vidWZTI&d-&V%|$~(6%N62!ND$@DGQh_z` ze`kMpZ#MpKxA*pUp5p&wv<*1(8F}#OQ)rw9

SuYcIVIxs*QCHXyvYv4C^{(JwAV zbfLPe2bPBDs#`88*8^z$w`w&WgaCQE@ONNB6h}#zVWRaSIFv{<0;zo9pkQnQzZZmA z2WgUIn_Wl2KhQ*Ho>+Jd`#X?~h*Z$WgcxV3M27IU%5~dbFrdg%1t!a}Q3X>QCroDa z-223%lb(S!vBLP}ttfai0AHp!BtB3nJD_&YsD>-bm~s8q!4i{a8R9*)=f>MkQxeME zpbkUgxlU7RJU(GlU+bbjUgsvf81k6lLsVs-)QzsRDRPt>ri8F>Z1EB8l4zY=XGP<7`EP=>UF6xGe|8=rLI-4Qx~0aQ6X;|l#xC%RFN?$UwlVDQ|cx<|~GFx&aGv%bd2CU{R zY7WciD>E!Oy#X!$Niuc^NQeAB^X}zV%i@-L({jifTxkU>h^s@E1eNj$VLsPe?M3fE zJo0hF<8_M>Z{PIJk54YUr)TE@xDwLpro`V1Dlvt|31YfHvcYOxrmxpTq2X$T63sJ{ zYu9U%#`4~O!0(07mAx;K;6h0OHNuZiUjOCo@zLes>D#x*hv%nfy&3w~$LZ`pYp%QXmd-VA1O~`zmoEf zw&p+^@CVc}`Z_j5uGiz51D(h9wdcKT zcd-AIU?#Z22Y(G(WB+@*JKOX2-)Xm>?Ef*^@_+?5H^$~x##E!hb__tRk@aDSFJLllhNA;;@x*-l`MFSHPM*%P@-QtAk1o-p7~a2+-W!;mq8*&p=C%m4Hj;iHcBrNf-)(Mppt5G~Y6Yhgddm$Fhv=)(LgPP`$-BB^1{*J{B!uXInWGcIba zG*=@};@hPP;P$H=QE9V|$*FY9_N&x*Ug}9%q}IoNSdBV0+M4-(?acgf+8ydY*(b?o z?E_j<|LyK{cILnTwV(F?JWl&E^`BJ{Qs|M8^j|jPk3PP%OY+k-B~~bn@=P%N9?L>m zFti)@NKSv2qR3SY7n_D<5-56ckG z%W@H~D*C@+qDhjVQuo523r3U4BF0>#3X?)oJ+UAZXAqipu~{gb>1jWancjDtMMF*) zeue`5h4Pg0;diM=g5ens48`Sy+`@eTuxLH^<<)D6)n13KH&b+fw(ypGE4{f{nBr<8 z=SKeyCvioEGr#-EfG*h)wX&*M=sz8Hd#<*a7p}AC*0|G0O-ey`McVVm+_mq&M_HH4 zTDhcS%5aEj3+HRo}j)G^5g8oH{HY82i-SE(+9HPuhrib`n5gd+GE%}Gf!Xh z$);WWdrNt7z50ET=L|FCnlSHF>!9xnr7ifTdrJ7Qp8sc*$Iz~_){I=ou)nvrvljpi zl=BH+2%SWdo^m!h6GGdM6t=vZiejcV31L%N&D$*hEz^7RfG{SmF#W54Wc!3R+3!jm z(CgJz${nDps$&**RT1$YoUni4c8C1mFVQ}{1Xz>*cRKqAi~IkczW+W-^P!CJ>1y!o z#`vYB!J0l*nQ+JQVLmnZuH#g>G+$env6eO6DqbE0c+3U)Tna4`!D2W+?D{hVa)`S6 zG_bm}y3>cGTa%?HU;VANyZe8CfaU>VVBPtzozDFJ@9n4apO4hO3I9JmVHqVtqEPlc z(R@_!$Y+AXXSwXhi!J}YO$F6Ry+zb+UB>6~_l$+ViN2*Qp6)ld!7V+vVZ&2x!(1I; zdV0*|@;16Gi-7L$GqkKxKCI|;OG>3wE-o((ZI_qjcOQvEypCb>Dx;UrqZeD7)$^Q@ z#Fk{{tv_}4tW&<%Q?g?yk=|>_5@)8=@9a`b!WBvE=7vl2dyJP_Xu@u#XJy|9km*Xd zCUBa}QEKG!C-bA)W04cT2Y7SoKq=a+%e`@g#U%&J_<7cOw*4U4CF-&~x0g$ipi)Ar zr4n<6>QZVmk>XQU@rByT_#cgsWpv1e#LpdL?f&1Lz4`OsI|onu{~o1nK$mEZ(j9$u z^KEJj4*i0%l=30?gU6~m{jG}JRym+X$QTTn=(~VJ`LGER88I%+F67jn6P^Yez;Wmf z1wGI08V{y$MfC{&Yb%0N&L$wZ?P*AY93?QyF$$ui-epeX!ZK$Pc3azhuk)EFX-9E%T%NSB-gVil|SLk40|JudCoiq^|PdMbsxBgBOv`%-=%N zVesPrf(`hENGb{iZ;p;t5am+*hKUX$nj&dAa4CKZqOnRuimmU#AFZ_iv*XuC?~bD^ z{roZ3*#FMn{Q1w$-rjcS$^IXsZLGdeBzRq(4fO{M`$S<1g4+{>7q=bvuaJCr{?Lm8 zw!oX0^%P@cc5DI(VWxh#%4j$Rw5Jo5zpaH#w;zKTlxNc&OhULj!jB90v)p#Ii{L=& zXFwD8F)lI^x~Sdxa_Wy-EoIsjHg(?1lUZBy_f;I8+1hG~`K(}QpGpM6l&DT&+QX*R z0njCvIv6}8P2Dk8iHVTbrQ5IB20kM>{920sS5PP^D(8{)2&shAUPy*qC`}Wv-|JB3 zL~46y#|E5;6yeORVP*CXfS^Ztf|USMfy!M@;9^GYGma@HRI^S$Ugwx-1q4jxl0ui} z2+x0A`oZAWEkizDQ>9CdEXN!PrQLknPZp}SL)eMPs+>LXTQX|46Q*o?BI&EgO2!CL#3Z}$EK!+jGjR$ckAZnUMHwwi{Mq`VWw zo=`4N2;044-ZZ>rOocL?eh20lX#!z_4;4*Ke_3#D#FB`7VyujSQ3{$Aj7ade4w>f{ zU(g?z>5-xs)=2Xl7$u(})dS2LI8{LU4ChrKwfoQ5WXkod9xb;igH@pc&!QlBGw}Cd zSifLzSuv(cxzdE0>b9=s?k)}jNujjJs!nzquE_L+^_UVk``dBH5mpk_^LFG{Qkw26 z9p|U~Icvfsl5_$iGRFC7ENAXdjw0W~LiQ24@fGgad2UG^z=c837eXs7Np9~x@s*ps z8ib^&V-IJ1CWz{mAIpMg@>WGIfA^W_w=!bjTV9t-WoT7|M7ip0MgEGOnTvyZh?T9@ zt`EC)Xge{gr6po(I0|!{v=6U)wdGuEYUKH1YctZ1%LsH-lhMaQhP6?%c52 zuN!=FAC`zD89cAHA6(c6s8K$DM|eu?rtnwr%^#Ly2 z1r#5ib;~WrTY>#W2k`tHl@84da6c{37JQoSW*@-w(4-p6LEYcylN_PiGrv{uZ`uYV z42cWbie$cqHI?BE*coPGjNu!N8H8t;6jC9CKM9%O?2t9#!W&mx{{g3k4o?ckDwH=t z?eu{zBoAs&Pw>hQog8h=0P#=8eE ze}wbHZv3+SvfT)tnYyG~^chtn)BpLVdH1P(vkBp#j(jT!45)E8rsQ+CFhCgQzHc4T dx14J~wWs#fp4zw6{yzW!|NpUW1ib)Y0032(0#5({ literal 0 HcmV?d00001 diff --git a/charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/Chart.yaml b/charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/Chart.yaml new file mode 100755 index 000000000..226e97e5f --- /dev/null +++ b/charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/Chart.yaml @@ -0,0 +1,10 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cis-operator-system + catalog.cattle.io/release-name: rancher-cis-benchmark-crd +apiVersion: v1 +description: Installs the CRDs for rancher-cis-benchmark. +name: rancher-cis-benchmark-crd +type: application +version: 1.0.400-rc00 diff --git a/charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/README.md b/charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/README.md new file mode 100755 index 000000000..f6d9ef621 --- /dev/null +++ b/charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/README.md @@ -0,0 +1,2 @@ +# rancher-cis-benchmark-crd +A Rancher chart that installs the CRDs used by rancher-cis-benchmark. diff --git a/charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/templates/clusterscan.yaml b/charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/templates/clusterscan.yaml new file mode 100755 index 000000000..beca6e1f8 --- /dev/null +++ b/charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/templates/clusterscan.yaml @@ -0,0 +1,149 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clusterscans.cis.cattle.io +spec: + additionalPrinterColumns: + - JSONPath: .status.lastRunScanProfileName + name: ClusterScanProfile + type: string + - JSONPath: .status.summary.total + name: Total + type: string + - JSONPath: .status.summary.pass + name: Pass + type: string + - JSONPath: .status.summary.fail + name: Fail + type: string + - JSONPath: .status.summary.skip + name: Skip + type: string + - JSONPath: .status.summary.warn + name: Warn + type: string + - JSONPath: .status.summary.notApplicable + name: Not Applicable + type: string + - JSONPath: .status.lastRunTimestamp + name: LastRunTimestamp + type: string + - JSONPath: .spec.scheduledScanConfig.cronSchedule + name: CronSchedule + type: string + group: cis.cattle.io + names: + kind: ClusterScan + plural: clusterscans + scope: Cluster + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + spec: + properties: + scanProfileName: + nullable: true + type: string + scheduledScanConfig: + nullable: true + properties: + cronSchedule: + nullable: true + type: string + retentionCount: + type: integer + scanAlertRule: + nullable: true + properties: + alertOnComplete: + type: boolean + alertOnFailure: + type: boolean + type: object + type: object + scoreWarning: + enum: + - pass + - fail + nullable: true + type: string + type: object + status: + properties: + NextScanAt: + nullable: true + type: string + ScanAlertingRuleName: + nullable: true + type: string + conditions: + items: + properties: + lastTransitionTime: + nullable: true + type: string + lastUpdateTime: + nullable: true + type: string + message: + nullable: true + type: string + reason: + nullable: true + type: string + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + display: + nullable: true + properties: + error: + type: boolean + message: + nullable: true + type: string + state: + nullable: true + type: string + transitioning: + type: boolean + type: object + lastRunScanProfileName: + nullable: true + type: string + lastRunTimestamp: + nullable: true + type: string + observedGeneration: + type: integer + summary: + nullable: true + properties: + fail: + type: integer + notApplicable: + type: integer + pass: + type: integer + skip: + type: integer + total: + type: integer + warn: + type: integer + type: object + type: object + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true diff --git a/charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/templates/clusterscanbenchmark.yaml b/charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/templates/clusterscanbenchmark.yaml new file mode 100755 index 000000000..aa6fc2218 --- /dev/null +++ b/charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/templates/clusterscanbenchmark.yaml @@ -0,0 +1,55 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clusterscanbenchmarks.cis.cattle.io +spec: + additionalPrinterColumns: + - JSONPath: .spec.clusterProvider + name: ClusterProvider + type: string + - JSONPath: .spec.minKubernetesVersion + name: MinKubernetesVersion + type: string + - JSONPath: .spec.maxKubernetesVersion + name: MaxKubernetesVersion + type: string + - JSONPath: .spec.customBenchmarkConfigMapName + name: customBenchmarkConfigMapName + type: string + - JSONPath: .spec.customBenchmarkConfigMapNamespace + name: customBenchmarkConfigMapNamespace + type: string + group: cis.cattle.io + names: + kind: ClusterScanBenchmark + plural: clusterscanbenchmarks + scope: Cluster + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + spec: + properties: + clusterProvider: + nullable: true + type: string + customBenchmarkConfigMapName: + nullable: true + type: string + customBenchmarkConfigMapNamespace: + nullable: true + type: string + maxKubernetesVersion: + nullable: true + type: string + minKubernetesVersion: + nullable: true + type: string + type: object + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true diff --git a/charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/templates/clusterscanprofile.yaml b/charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/templates/clusterscanprofile.yaml new file mode 100755 index 000000000..21bb68396 --- /dev/null +++ b/charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/templates/clusterscanprofile.yaml @@ -0,0 +1,37 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clusterscanprofiles.cis.cattle.io +spec: + additionalPrinterColumns: + - JSONPath: .spec.benchmarkVersion + name: BenchmarkVersion + type: string + group: cis.cattle.io + names: + kind: ClusterScanProfile + plural: clusterscanprofiles + scope: Cluster + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + spec: + properties: + benchmarkVersion: + nullable: true + type: string + skipTests: + items: + nullable: true + type: string + nullable: true + type: array + type: object + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true diff --git a/charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/templates/clusterscanreport.yaml b/charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/templates/clusterscanreport.yaml new file mode 100755 index 000000000..017020a95 --- /dev/null +++ b/charts/rancher-cis-benchmark/rancher-cis-benchmark-crd/1.0.400-rc00/templates/clusterscanreport.yaml @@ -0,0 +1,40 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clusterscanreports.cis.cattle.io +spec: + additionalPrinterColumns: + - JSONPath: .spec.lastRunTimestamp + name: LastRunTimestamp + type: string + - JSONPath: .spec.benchmarkVersion + name: BenchmarkVersion + type: string + group: cis.cattle.io + names: + kind: ClusterScanReport + plural: clusterscanreports + scope: Cluster + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + spec: + properties: + benchmarkVersion: + nullable: true + type: string + lastRunTimestamp: + nullable: true + type: string + reportJSON: + nullable: true + type: string + type: object + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true diff --git a/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/Chart.yaml b/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/Chart.yaml new file mode 100755 index 000000000..075d5aba1 --- /dev/null +++ b/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/Chart.yaml @@ -0,0 +1,10 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-logging-system + catalog.cattle.io/release-name: rancher-logging-crd +apiVersion: v1 +description: Installs the CRDs for rancher-logging. +name: rancher-logging-crd +type: application +version: 3.9.002-rc03 diff --git a/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/README.md b/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/README.md new file mode 100755 index 000000000..d4beb54fa --- /dev/null +++ b/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/README.md @@ -0,0 +1,2 @@ +# rancher-logging-crd +A Rancher chart that installs the CRDs used by rancher-logging. diff --git a/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_clusterflows.yaml b/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_clusterflows.yaml new file mode 100755 index 000000000..9fc6e22a5 --- /dev/null +++ b/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_clusterflows.yaml @@ -0,0 +1,765 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.4 + creationTimestamp: null + name: clusterflows.logging.banzaicloud.io +spec: + additionalPrinterColumns: + - JSONPath: .status.active + description: Is the flow active? + name: Active + type: boolean + - JSONPath: .status.problemsCount + description: Number of problems + name: Problems + type: integer + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: ClusterFlow + listKind: ClusterFlowList + plural: clusterflows + singular: clusterflow + preserveUnknownFields: false + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + filters: + items: + properties: + concat: + properties: + continuous_line_regexp: + type: string + flush_interval: + type: integer + keep_partial_key: + type: boolean + keep_partial_metadata: + type: string + key: + type: string + multiline_end_regexp: + type: string + multiline_start_regexp: + type: string + n_lines: + type: integer + partial_key: + type: string + partial_value: + type: string + separator: + type: string + stream_identity_key: + type: string + timeout_label: + type: string + use_first_timestamp: + type: boolean + use_partial_metadata: + type: string + type: object + dedot: + properties: + de_dot_nested: + type: boolean + de_dot_separator: + type: string + type: object + detectExceptions: + properties: + languages: + items: + type: string + type: array + max_bytes: + type: integer + max_lines: + type: integer + message: + type: string + multiline_flush_interval: + type: string + remove_tag_prefix: + type: string + stream: + type: string + type: object + enhanceK8s: + properties: + api_groups: + items: + type: string + type: array + bearer_token_file: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + cache_refresh: + type: integer + cache_refresh_variation: + type: integer + cache_size: + type: integer + cache_ttl: + type: integer + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + core_api_versions: + items: + type: string + type: array + data_type: + type: string + in_namespace_path: + items: + type: string + type: array + in_pod_path: + items: + type: string + type: array + kubernetes_url: + type: string + secret_dir: + type: string + ssl_partial_chain: + type: boolean + verify_ssl: + type: boolean + type: object + geoip: + properties: + backend_library: + type: string + geoip_2_database: + type: string + geoip_database: + type: string + geoip_lookup_keys: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + skip_adding_null_record: + type: boolean + type: object + grep: + properties: + and: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + or: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + parser: + properties: + emit_invalid_record_to_error: + type: boolean + hash_value_field: + type: string + inject_key_prefix: + type: string + key_name: + type: string + parse: + properties: + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + keep_time_key: + type: boolean + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + keep_time_key: + type: boolean + local_time: + type: boolean + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + parsers: + items: + properties: + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + keep_time_key: + type: boolean + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + keep_time_key: + type: boolean + local_time: + type: boolean + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + remove_key_name_field: + type: boolean + replace_invalid_sequence: + type: boolean + reserve_data: + type: boolean + reserve_time: + type: boolean + type: object + prometheus: + properties: + labels: + additionalProperties: + type: string + type: object + metrics: + items: + properties: + buckets: + type: string + desc: + type: string + key: + type: string + labels: + additionalProperties: + type: string + type: object + name: + type: string + type: + type: string + required: + - desc + - name + - type + type: object + type: array + type: object + record_modifier: + properties: + char_encoding: + type: string + prepare_value: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + replaces: + items: + properties: + expression: + type: string + key: + type: string + replace: + type: string + required: + - expression + - key + - replace + type: object + type: array + whitelist_keys: + type: string + type: object + record_transformer: + properties: + auto_typecast: + type: boolean + enable_ruby: + type: boolean + keep_keys: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + renew_record: + type: boolean + renew_time_key: + type: string + type: object + stdout: + properties: + output_type: + type: string + type: object + sumologic: + properties: + collector_key_name: + type: string + collector_value: + type: string + exclude_container_regex: + type: string + exclude_facility_regex: + type: string + exclude_host_regex: + type: string + exclude_namespace_regex: + type: string + exclude_pod_regex: + type: string + exclude_priority_regex: + type: string + exclude_unit_regex: + type: string + log_format: + type: string + source_category: + type: string + source_category_key_name: + type: string + source_category_prefix: + type: string + source_category_replace_dash: + type: string + source_host: + type: string + source_host_key_name: + type: string + source_name: + type: string + source_name_key_name: + type: string + tracing_annotation_prefix: + type: string + tracing_container_name: + type: string + tracing_format: + type: boolean + tracing_host: + type: string + tracing_label_prefix: + type: string + tracing_namespace: + type: string + tracing_pod: + type: string + tracing_pod_id: + type: string + type: object + tag_normaliser: + properties: + format: + type: string + type: object + throttle: + properties: + group_bucket_limit: + type: integer + group_bucket_period_s: + type: integer + group_drop_logs: + type: boolean + group_key: + type: string + group_reset_rate_s: + type: integer + group_warning_delay_s: + type: integer + type: object + type: object + type: array + globalOutputRefs: + items: + type: string + type: array + loggingRef: + type: string + match: + items: + properties: + exclude: + properties: + container_names: + items: + type: string + type: array + hosts: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + namespaces: + items: + type: string + type: array + type: object + select: + properties: + container_names: + items: + type: string + type: array + hosts: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + namespaces: + items: + type: string + type: array + type: object + type: object + type: array + outputRefs: + items: + type: string + type: array + selectors: + additionalProperties: + type: string + type: object + type: object + status: + properties: + active: + type: boolean + problems: + items: + type: string + type: array + problemsCount: + type: integer + type: object + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_clusteroutputs.yaml b/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_clusteroutputs.yaml new file mode 100755 index 000000000..a82252ad9 --- /dev/null +++ b/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_clusteroutputs.yaml @@ -0,0 +1,4563 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.4 + creationTimestamp: null + name: clusteroutputs.logging.banzaicloud.io +spec: + additionalPrinterColumns: + - JSONPath: .status.active + description: Is the output active? + name: Active + type: boolean + - JSONPath: .status.problemsCount + description: Number of problems + name: Problems + type: integer + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: ClusterOutput + listKind: ClusterOutputList + plural: clusteroutputs + singular: clusteroutput + preserveUnknownFields: false + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + awsElasticsearch: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + endpoint: + properties: + access_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + assume_role_arn: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + assume_role_session_name: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + assume_role_web_identity_token_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + ecs_container_credentials_relative_uri: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + region: + type: string + secret_access_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + sts_credentials_region: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + url: + type: string + type: object + flush_interval: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + include_tag_key: + type: boolean + logstash_format: + type: boolean + tag_key: + type: string + type: object + azurestorage: + properties: + auto_create_container: + type: boolean + azure_container: + type: string + azure_imds_api_version: + type: string + azure_object_key_format: + type: string + azure_storage_access_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + azure_storage_account: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + azure_storage_sas_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + type: string + path: + type: string + required: + - azure_container + - azure_storage_access_key + - azure_storage_account + - azure_storage_sas_token + type: object + cloudwatch: + properties: + auto_create_stream: + type: boolean + aws_instance_profile_credentials_retries: + type: integer + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + aws_sts_role_arn: + type: string + aws_sts_session_name: + type: string + aws_use_sts: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + concurrency: + type: integer + endpoint: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + http_proxy: + type: string + include_time_key: + type: boolean + json_handler: + type: string + localtime: + type: boolean + log_group_aws_tags: + type: string + log_group_aws_tags_key: + type: string + log_group_name: + type: string + log_group_name_key: + type: string + log_rejected_request: + type: string + log_stream_name: + type: string + log_stream_name_key: + type: string + max_events_per_batch: + type: integer + max_message_length: + type: integer + message_keys: + type: string + put_log_events_disable_retry_limit: + type: boolean + put_log_events_retry_limit: + type: integer + put_log_events_retry_wait: + type: string + region: + type: string + remove_log_group_aws_tags_key: + type: string + remove_log_group_name_key: + type: string + remove_log_stream_name_key: + type: string + remove_retention_in_days: + type: string + retention_in_days: + type: string + retention_in_days_key: + type: string + use_tag_as_group: + type: boolean + use_tag_as_stream: + type: boolean + required: + - region + type: object + datadog: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + compression_level: + type: string + dd_hostname: + type: string + dd_source: + type: string + dd_sourcecategory: + type: string + dd_tags: + type: string + host: + type: string + include_tag_key: + type: boolean + max_backoff: + type: string + max_retries: + type: string + no_ssl_validation: + type: boolean + port: + type: string + service: + type: string + ssl_port: + type: string + tag_key: + type: string + timestamp_key: + type: string + use_compression: + type: boolean + use_http: + type: boolean + use_json: + type: boolean + use_ssl: + type: boolean + required: + - api_key + type: object + elasticsearch: + properties: + application_name: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + bulk_message_request_threshold: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + client_key_pass: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + content_type: + type: string + custom_headers: + type: string + customize_template: + type: string + default_elasticsearch_version: + type: string + deflector_alias: + type: string + enable_ilm: + type: boolean + exception_backup: + type: boolean + fail_on_putting_template_retry_exceed: + type: boolean + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + hosts: + type: string + http_backend: + type: string + id_key: + type: string + ignore_exceptions: + type: string + ilm_policy: + type: string + ilm_policy_id: + type: string + ilm_policy_overwrite: + type: boolean + include_index_in_url: + type: boolean + include_tag_key: + type: boolean + include_timestamp: + type: boolean + index_date_pattern: + type: string + index_name: + type: string + index_prefix: + type: string + log_es_400_reason: + type: boolean + logstash_dateformat: + type: string + logstash_format: + type: boolean + logstash_prefix: + type: string + logstash_prefix_separator: + type: string + max_retry_get_es_version: + type: string + max_retry_putting_template: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + path: + type: string + pipeline: + type: string + port: + type: integer + prefer_oj_serializer: + type: boolean + reconnect_on_error: + type: boolean + reload_after: + type: string + reload_connections: + type: boolean + reload_on_failure: + type: boolean + remove_keys_on_update: + type: string + remove_keys_on_update_key: + type: string + request_timeout: + type: string + resurrect_after: + type: string + retry_tag: + type: string + rollover_index: + type: boolean + routing_key: + type: string + scheme: + type: string + sniffer_class_name: + type: string + ssl_max_version: + type: string + ssl_min_version: + type: string + ssl_verify: + type: boolean + ssl_version: + type: string + suppress_doc_wrap: + type: boolean + suppress_type_name: + type: boolean + tag_key: + type: string + target_index_key: + type: string + target_type_key: + type: string + template_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + template_name: + type: string + template_overwrite: + type: boolean + templates: + type: string + time_key: + type: string + time_key_format: + type: string + time_parse_error_tag: + type: string + time_precision: + type: string + type_name: + type: string + unrecoverable_error_types: + type: string + user: + type: string + utc_index: + type: boolean + validate_client_version: + type: boolean + verify_es_version_at_startup: + type: boolean + with_transporter_log: + type: boolean + write_operation: + type: string + type: object + enabledNamespaces: + items: + type: string + type: array + file: + properties: + add_path_suffix: + type: boolean + append: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + path: + type: string + path_suffix: + type: string + symlink_path: + type: boolean + required: + - path + type: object + forward: + properties: + ack_response_timeout: + type: integer + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + connect_timeout: + type: integer + dns_round_robin: + type: boolean + expire_dns_cache: + type: integer + hard_timeout: + type: integer + heartbeat_interval: + type: integer + heartbeat_type: + type: string + ignore_network_errors_at_startup: + type: boolean + keepalive: + type: boolean + keepalive_timeout: + type: integer + phi_failure_detector: + type: boolean + phi_threshold: + type: integer + recover_wait: + type: integer + require_ack_response: + type: boolean + security: + properties: + allow_anonymous_source: + type: boolean + self_hostname: + type: string + shared_key: + type: string + user_auth: + type: boolean + required: + - self_hostname + - shared_key + type: object + send_timeout: + type: integer + servers: + items: + properties: + host: + type: string + name: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + port: + type: integer + shared_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + standby: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + weight: + type: integer + required: + - host + type: object + type: array + tls_allow_self_signed_cert: + type: boolean + tls_cert_logical_store_name: + type: string + tls_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + tls_cert_thumbprint: + type: string + tls_cert_use_enterprise_store: + type: boolean + tls_ciphers: + type: string + tls_client_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + tls_client_private_key_passphrase: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + tls_client_private_key_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + tls_insecure_mode: + type: boolean + tls_verify_hostname: + type: boolean + tls_version: + type: string + verify_connection_at_startup: + type: boolean + required: + - servers + type: object + gcs: + properties: + acl: + type: string + auto_create_bucket: + type: boolean + bucket: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + client_retries: + type: integer + client_timeout: + type: integer + credentials_json: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + encryption_key: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + hex_random_length: + type: integer + keyfile: + type: string + object_key_format: + type: string + object_metadata: + items: + properties: + key: + type: string + value: + type: string + required: + - key + - value + type: object + type: array + overwrite: + type: boolean + path: + type: string + project: + type: string + storage_class: + type: string + store_as: + type: string + transcoding: + type: boolean + required: + - bucket + - project + type: object + http: + properties: + auth: + properties: + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + required: + - password + - username + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + content_type: + type: string + endpoint: + type: string + error_response_as_unrecoverable: + type: boolean + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + headers: + additionalProperties: + type: string + type: object + http_method: + type: string + json_array: + type: boolean + open_timeout: + type: integer + proxy: + type: string + read_timeout: + type: integer + retryable_response_codes: + items: + type: integer + type: array + ssl_timeout: + type: integer + tls_ca_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + tls_ciphers: + type: string + tls_client_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + tls_private_key_passphrase: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + tls_private_key_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + tls_verify_mode: + type: string + tls_version: + type: string + required: + - endpoint + type: object + kafka: + properties: + ack_timeout: + type: integer + brokers: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + compression_codec: + type: string + default_message_key: + type: string + default_partition_key: + type: string + default_topic: + type: string + exclude_partion_key: + type: boolean + exclude_topic_key: + type: boolean + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + get_kafka_client_log: + type: boolean + headers: + additionalProperties: + type: string + type: object + headers_from_record: + additionalProperties: + type: string + type: object + idempotent: + type: boolean + max_send_retries: + type: integer + message_key_key: + type: string + partition_key: + type: string + partition_key_key: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + required_acks: + type: integer + sasl_over_ssl: + type: boolean + scram_mechanism: + type: string + ssl_ca_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + ssl_ca_certs_from_system: + type: boolean + ssl_client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + ssl_client_cert_chain: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + ssl_client_cert_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + ssl_verify_hostname: + type: boolean + topic_key: + type: string + use_default_for_unknown_topic: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + required: + - brokers + - format + type: object + kinesisStream: + properties: + assume_role_credentials: + properties: + duration_seconds: + type: string + external_id: + type: string + policy: + type: string + role_arn: + type: string + role_session_name: + type: string + required: + - role_arn + - role_session_name + type: object + aws_iam_retries: + type: integer + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + aws_ses_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + batch_request_max_count: + type: integer + batch_request_max_size: + type: integer + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + partition_key: + type: string + process_credentials: + properties: + process: + type: string + required: + - process + type: object + region: + type: string + reset_backoff_if_success: + type: boolean + retries_on_batch_request: + type: integer + stream_name: + type: string + required: + - stream_name + type: object + logdna: + properties: + api_key: + type: string + app: + type: string + buffer_chunk_limit: + type: string + hostname: + type: string + required: + - api_key + - hostname + type: object + loggingRef: + type: string + logz: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + endpoint: + properties: + port: + type: integer + token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + url: + type: string + type: object + gzip: + type: boolean + http_idle_timeout: + type: integer + output_include_tags: + type: boolean + output_include_time: + type: boolean + retry_count: + type: integer + retry_sleep: + type: integer + required: + - endpoint + type: object + loki: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + configure_kubernetes_labels: + type: boolean + drop_single_key: + type: boolean + extra_labels: + additionalProperties: + type: string + type: object + extract_kubernetes_labels: + type: boolean + insecure_tls: + type: boolean + key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + labels: + additionalProperties: + type: string + type: object + line_format: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + remove_keys: + items: + type: string + type: array + tenant: + type: string + url: + type: string + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + type: object + newrelic: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + base_uri: + type: string + license_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + type: object + nullout: + type: object + oss: + properties: + aaccess_key_secret: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + access_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + auto_create_bucket: + type: boolean + bucket: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + check_bucket: + type: boolean + check_object: + type: boolean + download_crc_enable: + type: boolean + endpoint: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + hex_random_length: + type: integer + index_format: + type: string + key_format: + type: string + open_timeout: + type: integer + oss_sdk_log_dir: + type: string + overwrite: + type: boolean + path: + type: string + read_timeout: + type: integer + store_as: + type: string + upload_crc_enable: + type: boolean + warn_for_delay: + type: string + required: + - aaccess_key_secret + - access_key_id + - bucket + - endpoint + type: object + redis: + properties: + allow_duplicate_key: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + db_number: + type: integer + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + host: + type: string + insert_key_prefix: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + port: + type: integer + strftime_format: + type: string + ttl: + type: integer + type: object + s3: + properties: + acl: + type: string + assume_role_credentials: + properties: + duration_seconds: + type: string + external_id: + type: string + policy: + type: string + role_arn: + type: string + role_session_name: + type: string + required: + - role_arn + - role_session_name + type: object + auto_create_bucket: + type: string + aws_iam_retries: + type: string + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + check_apikey_on_start: + type: string + check_bucket: + type: string + check_object: + type: string + clustername: + type: string + compute_checksums: + type: string + enable_transfer_acceleration: + type: string + force_path_style: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + grant_full_control: + type: string + grant_read: + type: string + grant_read_acp: + type: string + grant_write_acp: + type: string + hex_random_length: + type: string + index_format: + type: string + instance_profile_credentials: + properties: + http_open_timeout: + type: string + http_read_timeout: + type: string + ip_address: + type: string + port: + type: string + retries: + type: string + type: object + oneeye_format: + type: boolean + overwrite: + type: string + path: + type: string + proxy_uri: + type: string + s3_bucket: + type: string + s3_endpoint: + type: string + s3_metadata: + type: string + s3_object_key_format: + type: string + s3_region: + type: string + shared_credentials: + properties: + path: + type: string + profile_name: + type: string + type: object + signature_version: + type: string + sse_customer_algorithm: + type: string + sse_customer_key: + type: string + sse_customer_key_md5: + type: string + ssekms_key_id: + type: string + ssl_verify_peer: + type: string + storage_class: + type: string + store_as: + type: string + use_bundled_cert: + type: string + use_server_side_encryption: + type: string + warn_for_delay: + type: string + required: + - s3_bucket + type: object + splunkHec: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + ca_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + coerce_to_utf8: + type: boolean + data_type: + type: string + fields: + additionalProperties: + type: string + type: object + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + hec_host: + type: string + hec_port: + type: integer + hec_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + host: + type: string + host_key: + type: string + idle_timeout: + type: integer + index: + type: string + index_key: + type: string + insecure_ssl: + type: boolean + keep_keys: + type: boolean + metric_name_key: + type: string + metric_value_key: + type: string + metrics_from_event: + type: boolean + non_utf8_replacement_string: + type: string + open_timeout: + type: integer + protocol: + type: string + read_timeout: + type: integer + source: + type: string + source_key: + type: string + sourcetype: + type: string + sourcetype_key: + type: string + ssl_ciphers: + type: string + required: + - hec_host + - hec_token + type: object + sumologic: + properties: + add_timestamp: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + compress: + type: boolean + compress_encoding: + type: string + custom_dimensions: + type: string + custom_fields: + items: + type: string + type: array + data_type: + type: string + delimiter: + type: string + disable_cookies: + type: boolean + endpoint: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + log_format: + type: string + log_key: + type: string + metric_data_format: + type: string + open_timeout: + type: integer + proxy_uri: + type: string + source_category: + type: string + source_host: + type: string + source_name: + type: string + source_name_key: + type: string + sumo_client: + type: string + timestamp_key: + type: string + verify_ssl: + type: boolean + required: + - endpoint + - source_name + type: object + syslog: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + properties: + app_name_field: + type: string + hostname_field: + type: string + log_field: + type: string + message_id_field: + type: string + proc_id_field: + type: string + rfc6587_message_size: + type: boolean + structured_data_field: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + host: + type: string + insecure: + type: boolean + port: + type: integer + transport: + type: string + trusted_ca_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + required: + - host + type: object + type: object + status: + properties: + active: + type: boolean + problems: + items: + type: string + type: array + problemsCount: + type: integer + type: object + required: + - spec + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_flows.yaml b/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_flows.yaml new file mode 100755 index 000000000..a01a1331d --- /dev/null +++ b/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_flows.yaml @@ -0,0 +1,761 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.4 + creationTimestamp: null + name: flows.logging.banzaicloud.io +spec: + additionalPrinterColumns: + - JSONPath: .status.active + description: Is the flow active? + name: Active + type: boolean + - JSONPath: .status.problemsCount + description: Number of problems + name: Problems + type: integer + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: Flow + listKind: FlowList + plural: flows + singular: flow + preserveUnknownFields: false + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + filters: + items: + properties: + concat: + properties: + continuous_line_regexp: + type: string + flush_interval: + type: integer + keep_partial_key: + type: boolean + keep_partial_metadata: + type: string + key: + type: string + multiline_end_regexp: + type: string + multiline_start_regexp: + type: string + n_lines: + type: integer + partial_key: + type: string + partial_value: + type: string + separator: + type: string + stream_identity_key: + type: string + timeout_label: + type: string + use_first_timestamp: + type: boolean + use_partial_metadata: + type: string + type: object + dedot: + properties: + de_dot_nested: + type: boolean + de_dot_separator: + type: string + type: object + detectExceptions: + properties: + languages: + items: + type: string + type: array + max_bytes: + type: integer + max_lines: + type: integer + message: + type: string + multiline_flush_interval: + type: string + remove_tag_prefix: + type: string + stream: + type: string + type: object + enhanceK8s: + properties: + api_groups: + items: + type: string + type: array + bearer_token_file: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + cache_refresh: + type: integer + cache_refresh_variation: + type: integer + cache_size: + type: integer + cache_ttl: + type: integer + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + core_api_versions: + items: + type: string + type: array + data_type: + type: string + in_namespace_path: + items: + type: string + type: array + in_pod_path: + items: + type: string + type: array + kubernetes_url: + type: string + secret_dir: + type: string + ssl_partial_chain: + type: boolean + verify_ssl: + type: boolean + type: object + geoip: + properties: + backend_library: + type: string + geoip_2_database: + type: string + geoip_database: + type: string + geoip_lookup_keys: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + skip_adding_null_record: + type: boolean + type: object + grep: + properties: + and: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + or: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + parser: + properties: + emit_invalid_record_to_error: + type: boolean + hash_value_field: + type: string + inject_key_prefix: + type: string + key_name: + type: string + parse: + properties: + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + keep_time_key: + type: boolean + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + keep_time_key: + type: boolean + local_time: + type: boolean + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + parsers: + items: + properties: + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + keep_time_key: + type: boolean + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + keep_time_key: + type: boolean + local_time: + type: boolean + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + remove_key_name_field: + type: boolean + replace_invalid_sequence: + type: boolean + reserve_data: + type: boolean + reserve_time: + type: boolean + type: object + prometheus: + properties: + labels: + additionalProperties: + type: string + type: object + metrics: + items: + properties: + buckets: + type: string + desc: + type: string + key: + type: string + labels: + additionalProperties: + type: string + type: object + name: + type: string + type: + type: string + required: + - desc + - name + - type + type: object + type: array + type: object + record_modifier: + properties: + char_encoding: + type: string + prepare_value: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + replaces: + items: + properties: + expression: + type: string + key: + type: string + replace: + type: string + required: + - expression + - key + - replace + type: object + type: array + whitelist_keys: + type: string + type: object + record_transformer: + properties: + auto_typecast: + type: boolean + enable_ruby: + type: boolean + keep_keys: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + renew_record: + type: boolean + renew_time_key: + type: string + type: object + stdout: + properties: + output_type: + type: string + type: object + sumologic: + properties: + collector_key_name: + type: string + collector_value: + type: string + exclude_container_regex: + type: string + exclude_facility_regex: + type: string + exclude_host_regex: + type: string + exclude_namespace_regex: + type: string + exclude_pod_regex: + type: string + exclude_priority_regex: + type: string + exclude_unit_regex: + type: string + log_format: + type: string + source_category: + type: string + source_category_key_name: + type: string + source_category_prefix: + type: string + source_category_replace_dash: + type: string + source_host: + type: string + source_host_key_name: + type: string + source_name: + type: string + source_name_key_name: + type: string + tracing_annotation_prefix: + type: string + tracing_container_name: + type: string + tracing_format: + type: boolean + tracing_host: + type: string + tracing_label_prefix: + type: string + tracing_namespace: + type: string + tracing_pod: + type: string + tracing_pod_id: + type: string + type: object + tag_normaliser: + properties: + format: + type: string + type: object + throttle: + properties: + group_bucket_limit: + type: integer + group_bucket_period_s: + type: integer + group_drop_logs: + type: boolean + group_key: + type: string + group_reset_rate_s: + type: integer + group_warning_delay_s: + type: integer + type: object + type: object + type: array + globalOutputRefs: + items: + type: string + type: array + localOutputRefs: + items: + type: string + type: array + loggingRef: + type: string + match: + items: + properties: + exclude: + properties: + container_names: + items: + type: string + type: array + hosts: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + type: object + select: + properties: + container_names: + items: + type: string + type: array + hosts: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + type: object + type: object + type: array + outputRefs: + items: + type: string + type: array + selectors: + additionalProperties: + type: string + type: object + type: object + status: + properties: + active: + type: boolean + problems: + items: + type: string + type: array + problemsCount: + type: integer + type: object + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_loggings.yaml b/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_loggings.yaml new file mode 100755 index 000000000..d14c80e2c --- /dev/null +++ b/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_loggings.yaml @@ -0,0 +1,3536 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.4 + creationTimestamp: null + name: loggings.logging.banzaicloud.io +spec: + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: Logging + listKind: LoggingList + plural: loggings + singular: logging + preserveUnknownFields: false + scope: Cluster + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + allowClusterResourcesFromAllNamespaces: + type: boolean + controlNamespace: + type: string + defaultFlow: + properties: + filters: + items: + properties: + concat: + properties: + continuous_line_regexp: + type: string + flush_interval: + type: integer + keep_partial_key: + type: boolean + keep_partial_metadata: + type: string + key: + type: string + multiline_end_regexp: + type: string + multiline_start_regexp: + type: string + n_lines: + type: integer + partial_key: + type: string + partial_value: + type: string + separator: + type: string + stream_identity_key: + type: string + timeout_label: + type: string + use_first_timestamp: + type: boolean + use_partial_metadata: + type: string + type: object + dedot: + properties: + de_dot_nested: + type: boolean + de_dot_separator: + type: string + type: object + detectExceptions: + properties: + languages: + items: + type: string + type: array + max_bytes: + type: integer + max_lines: + type: integer + message: + type: string + multiline_flush_interval: + type: string + remove_tag_prefix: + type: string + stream: + type: string + type: object + enhanceK8s: + properties: + api_groups: + items: + type: string + type: array + bearer_token_file: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + cache_refresh: + type: integer + cache_refresh_variation: + type: integer + cache_size: + type: integer + cache_ttl: + type: integer + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + core_api_versions: + items: + type: string + type: array + data_type: + type: string + in_namespace_path: + items: + type: string + type: array + in_pod_path: + items: + type: string + type: array + kubernetes_url: + type: string + secret_dir: + type: string + ssl_partial_chain: + type: boolean + verify_ssl: + type: boolean + type: object + geoip: + properties: + backend_library: + type: string + geoip_2_database: + type: string + geoip_database: + type: string + geoip_lookup_keys: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + skip_adding_null_record: + type: boolean + type: object + grep: + properties: + and: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + or: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + parser: + properties: + emit_invalid_record_to_error: + type: boolean + hash_value_field: + type: string + inject_key_prefix: + type: string + key_name: + type: string + parse: + properties: + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + keep_time_key: + type: boolean + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + keep_time_key: + type: boolean + local_time: + type: boolean + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + parsers: + items: + properties: + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + keep_time_key: + type: boolean + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + keep_time_key: + type: boolean + local_time: + type: boolean + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + remove_key_name_field: + type: boolean + replace_invalid_sequence: + type: boolean + reserve_data: + type: boolean + reserve_time: + type: boolean + type: object + prometheus: + properties: + labels: + additionalProperties: + type: string + type: object + metrics: + items: + properties: + buckets: + type: string + desc: + type: string + key: + type: string + labels: + additionalProperties: + type: string + type: object + name: + type: string + type: + type: string + required: + - desc + - name + - type + type: object + type: array + type: object + record_modifier: + properties: + char_encoding: + type: string + prepare_value: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + replaces: + items: + properties: + expression: + type: string + key: + type: string + replace: + type: string + required: + - expression + - key + - replace + type: object + type: array + whitelist_keys: + type: string + type: object + record_transformer: + properties: + auto_typecast: + type: boolean + enable_ruby: + type: boolean + keep_keys: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + renew_record: + type: boolean + renew_time_key: + type: string + type: object + stdout: + properties: + output_type: + type: string + type: object + sumologic: + properties: + collector_key_name: + type: string + collector_value: + type: string + exclude_container_regex: + type: string + exclude_facility_regex: + type: string + exclude_host_regex: + type: string + exclude_namespace_regex: + type: string + exclude_pod_regex: + type: string + exclude_priority_regex: + type: string + exclude_unit_regex: + type: string + log_format: + type: string + source_category: + type: string + source_category_key_name: + type: string + source_category_prefix: + type: string + source_category_replace_dash: + type: string + source_host: + type: string + source_host_key_name: + type: string + source_name: + type: string + source_name_key_name: + type: string + tracing_annotation_prefix: + type: string + tracing_container_name: + type: string + tracing_format: + type: boolean + tracing_host: + type: string + tracing_label_prefix: + type: string + tracing_namespace: + type: string + tracing_pod: + type: string + tracing_pod_id: + type: string + type: object + tag_normaliser: + properties: + format: + type: string + type: object + throttle: + properties: + group_bucket_limit: + type: integer + group_bucket_period_s: + type: integer + group_drop_logs: + type: boolean + group_key: + type: string + group_reset_rate_s: + type: integer + group_warning_delay_s: + type: integer + type: object + type: object + type: array + globalOutputRefs: + items: + type: string + type: array + outputRefs: + items: + type: string + type: array + type: object + enableRecreateWorkloadOnImmutableFieldChange: + type: boolean + flowConfigCheckDisabled: + type: boolean + flowConfigOverride: + type: string + fluentbit: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + bufferStorage: + properties: + storage.backlog.mem_limit: + type: string + storage.checksum: + type: string + storage.path: + type: string + storage.sync: + type: string + type: object + bufferStorageVolume: + properties: + emptyDir: + properties: + medium: + type: string + sizeLimit: + type: string + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + type: string + type: object + requests: + additionalProperties: + type: string + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + type: object + coroStackSize: + format: int32 + type: integer + customConfigSecret: + type: string + disableKubernetesFilter: + type: boolean + enableUpstream: + type: boolean + extraVolumeMounts: + items: + properties: + destination: + pattern: ^/.+$ + type: string + readOnly: + type: boolean + source: + pattern: ^/.+$ + type: string + required: + - destination + - source + type: object + type: array + filterAws: + properties: + Match: + type: string + account_id: + type: boolean + ami_id: + type: boolean + az: + type: boolean + ec2_instance_id: + type: boolean + ec2_instance_type: + type: boolean + hostname: + type: boolean + imds_version: + type: string + private_ip: + type: boolean + vpc_id: + type: boolean + type: object + filterKubernetes: + properties: + Annotations: + type: string + Buffer_Size: + type: string + Dummy_Meta: + type: string + K8S-Logging.Exclude: + type: string + K8S-Logging.Parser: + type: string + Keep_Log: + type: string + Kube_CA_File: + type: string + Kube_CA_Path: + type: string + Kube_Tag_Prefix: + type: string + Kube_Token_File: + type: string + Kube_URL: + type: string + Kube_meta_preload_cache_dir: + type: string + Labels: + type: string + Match: + type: string + Merge_Log: + type: string + Merge_Log_Key: + type: string + Merge_Log_Trim: + type: string + Merge_Parser: + type: string + Regex_Parser: + type: string + Use_Journal: + type: string + tls.debug: + type: string + tls.verify: + type: string + type: object + flush: + format: int32 + type: integer + forwardOptions: + properties: + Require_ack_response: + type: boolean + Retry_Limit: + type: string + Send_options: + type: boolean + Tag: + type: string + Time_as_Integer: + type: boolean + type: object + grace: + format: int32 + type: integer + image: + properties: + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + inputTail: + properties: + Buffer_Chunk_Size: + type: string + Buffer_Max_Size: + type: string + DB: + type: string + DB_Sync: + type: string + Docker_Mode: + type: string + Docker_Mode_Flush: + type: string + Exclude_Path: + type: string + Ignore_Older: + type: string + Key: + type: string + Mem_Buf_Limit: + type: string + Multiline: + type: string + Multiline_Flush: + type: string + Parser: + type: string + Parser_Firstline: + type: string + Parser_N: + items: + type: string + type: array + Path: + type: string + Path_Key: + type: string + Refresh_Interval: + type: string + Rotate_Wait: + type: string + Skip_Long_Lines: + type: string + Tag: + type: string + Tag_Regex: + type: string + storage.type: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + livenessDefaultCheck: + type: boolean + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + format: int32 + type: integer + type: object + logLevel: + type: string + metrics: + properties: + interval: + type: string + path: + type: string + port: + format: int32 + type: integer + prometheusAnnotations: + type: boolean + serviceMonitor: + type: boolean + serviceMonitorConfig: + properties: + additionalLabels: + additionalProperties: + type: string + type: object + honorLabels: + type: boolean + metricRelabelings: + items: + properties: + action: + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + type: string + type: array + targetLabel: + type: string + type: object + type: array + relabelings: + items: + properties: + action: + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + type: string + type: array + targetLabel: + type: string + type: object + type: array + type: object + timeout: + type: string + type: object + mountPath: + type: string + network: + properties: + connectTimeout: + format: int32 + type: integer + keepalive: + type: boolean + keepaliveIdleTimeout: + format: int32 + type: integer + keepaliveMaxRecycle: + format: int32 + type: integer + type: object + nodeSelector: + additionalProperties: + type: string + type: object + parser: + type: string + podPriorityClassName: + type: string + position_db: + properties: + emptyDir: + properties: + medium: + type: string + sizeLimit: + type: string + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + type: string + type: object + requests: + additionalProperties: + type: string + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + type: object + positiondb: + properties: + emptyDir: + properties: + medium: + type: string + sizeLimit: + type: string + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + type: string + type: object + requests: + additionalProperties: + type: string + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + type: object + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + format: int32 + type: integer + type: object + resources: + properties: + limits: + additionalProperties: + type: string + type: object + requests: + additionalProperties: + type: string + type: object + type: object + security: + properties: + podSecurityContext: + properties: + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + podSecurityPolicyCreate: + type: boolean + roleBasedAccessControlCreate: + type: boolean + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + capabilities: + properties: + add: + items: + type: string + type: array + drop: + items: + type: string + type: array + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + serviceAccount: + type: string + type: object + targetHost: + type: string + targetPort: + format: int32 + type: integer + tls: + properties: + enabled: + type: boolean + secretName: + type: string + sharedKey: + type: string + required: + - enabled + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + fluentd: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + bufferStorageVolume: + properties: + emptyDir: + properties: + medium: + type: string + sizeLimit: + type: string + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + type: string + type: object + requests: + additionalProperties: + type: string + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + type: object + configCheckAnnotations: + additionalProperties: + type: string + type: object + configReloaderImage: + properties: + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + disablePvc: + type: boolean + fluentLogDestination: + type: string + fluentOutLogrotate: + properties: + age: + type: string + enabled: + type: boolean + path: + type: string + size: + type: string + required: + - enabled + type: object + fluentdPvcSpec: + properties: + emptyDir: + properties: + medium: + type: string + sizeLimit: + type: string + type: object + host_path: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + pvc: + properties: + source: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + type: string + type: object + requests: + additionalProperties: + type: string + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + storageClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + type: object + type: object + forwardInputConfig: + properties: + add_tag_prefix: + type: string + bind: + type: string + chunk_size_limit: + type: string + chunk_size_warn_limit: + type: string + deny_keepalive: + type: boolean + linger_timeout: + type: integer + port: + type: string + resolve_hostname: + type: boolean + security: + properties: + allow_anonymous_source: + type: boolean + self_hostname: + type: string + shared_key: + type: string + user_auth: + type: boolean + required: + - self_hostname + - shared_key + type: object + send_keepalive_packet: + type: boolean + skip_invalid_event: + type: boolean + source_address_key: + type: string + sourceHostnameKey: + type: string + tag: + type: string + transport: + properties: + ca_cert_path: + type: string + ca_path: + type: string + ca_private_key_passphrase: + type: string + ca_private_key_path: + type: string + cert_path: + type: string + ciphers: + type: string + client_cert_auth: + type: boolean + insecure: + type: boolean + private_key_passphrase: + type: string + private_key_path: + type: string + protocol: + type: string + version: + type: string + type: object + type: object + ignoreRepeatedLogInterval: + type: string + ignoreSameLogInterval: + type: string + image: + properties: + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + livenessDefaultCheck: + type: boolean + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + format: int32 + type: integer + type: object + logLevel: + type: string + metrics: + properties: + interval: + type: string + path: + type: string + port: + format: int32 + type: integer + prometheusAnnotations: + type: boolean + serviceMonitor: + type: boolean + serviceMonitorConfig: + properties: + additionalLabels: + additionalProperties: + type: string + type: object + honorLabels: + type: boolean + metricRelabelings: + items: + properties: + action: + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + type: string + type: array + targetLabel: + type: string + type: object + type: array + relabelings: + items: + properties: + action: + type: string + modulus: + format: int64 + type: integer + regex: + type: string + replacement: + type: string + separator: + type: string + sourceLabels: + items: + type: string + type: array + targetLabel: + type: string + type: object + type: array + type: object + timeout: + type: string + type: object + nodeSelector: + additionalProperties: + type: string + type: object + podPriorityClassName: + type: string + port: + format: int32 + type: integer + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + type: object + failureThreshold: + format: int32 + type: integer + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + format: int32 + type: integer + type: object + resources: + properties: + limits: + additionalProperties: + type: string + type: object + requests: + additionalProperties: + type: string + type: object + type: object + rootDir: + type: string + scaling: + properties: + podManagementPolicy: + type: string + replicas: + type: integer + type: object + security: + properties: + podSecurityContext: + properties: + fsGroup: + format: int64 + type: integer + fsGroupChangePolicy: + type: string + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + supplementalGroups: + items: + format: int64 + type: integer + type: array + sysctls: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + podSecurityPolicyCreate: + type: boolean + roleBasedAccessControlCreate: + type: boolean + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + capabilities: + properties: + add: + items: + type: string + type: array + drop: + items: + type: string + type: array + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + runAsUserName: + type: string + type: object + type: object + serviceAccount: + type: string + type: object + tls: + properties: + enabled: + type: boolean + secretName: + type: string + sharedKey: + type: string + required: + - enabled + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + volumeModImage: + properties: + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + pullPolicy: + type: string + repository: + type: string + tag: + type: string + type: object + volumeMountChmod: + type: boolean + workers: + format: int32 + type: integer + type: object + globalFilters: + items: + properties: + concat: + properties: + continuous_line_regexp: + type: string + flush_interval: + type: integer + keep_partial_key: + type: boolean + keep_partial_metadata: + type: string + key: + type: string + multiline_end_regexp: + type: string + multiline_start_regexp: + type: string + n_lines: + type: integer + partial_key: + type: string + partial_value: + type: string + separator: + type: string + stream_identity_key: + type: string + timeout_label: + type: string + use_first_timestamp: + type: boolean + use_partial_metadata: + type: string + type: object + dedot: + properties: + de_dot_nested: + type: boolean + de_dot_separator: + type: string + type: object + detectExceptions: + properties: + languages: + items: + type: string + type: array + max_bytes: + type: integer + max_lines: + type: integer + message: + type: string + multiline_flush_interval: + type: string + remove_tag_prefix: + type: string + stream: + type: string + type: object + enhanceK8s: + properties: + api_groups: + items: + type: string + type: array + bearer_token_file: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + cache_refresh: + type: integer + cache_refresh_variation: + type: integer + cache_size: + type: integer + cache_ttl: + type: integer + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + core_api_versions: + items: + type: string + type: array + data_type: + type: string + in_namespace_path: + items: + type: string + type: array + in_pod_path: + items: + type: string + type: array + kubernetes_url: + type: string + secret_dir: + type: string + ssl_partial_chain: + type: boolean + verify_ssl: + type: boolean + type: object + geoip: + properties: + backend_library: + type: string + geoip_2_database: + type: string + geoip_database: + type: string + geoip_lookup_keys: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + skip_adding_null_record: + type: boolean + type: object + grep: + properties: + and: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + or: + items: + properties: + exclude: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + type: array + regexp: + items: + properties: + key: + type: string + pattern: + type: string + required: + - key + - pattern + type: object + type: array + type: object + parser: + properties: + emit_invalid_record_to_error: + type: boolean + hash_value_field: + type: string + inject_key_prefix: + type: string + key_name: + type: string + parse: + properties: + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + keep_time_key: + type: boolean + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + keep_time_key: + type: boolean + local_time: + type: boolean + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + parsers: + items: + properties: + delimiter: + type: string + delimiter_pattern: + type: string + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + format_firstline: + type: string + keep_time_key: + type: boolean + label_delimiter: + type: string + local_time: + type: boolean + multiline: + items: + type: string + type: array + null_empty_string: + type: boolean + null_value_pattern: + type: string + patterns: + items: + properties: + estimate_current_event: + type: boolean + expression: + type: string + format: + type: string + keep_time_key: + type: boolean + local_time: + type: boolean + null_empty_string: + type: boolean + null_value_pattern: + type: string + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + time_format: + type: string + time_key: + type: string + time_type: + type: string + timezone: + type: string + type: + type: string + types: + type: string + utc: + type: boolean + type: object + type: array + remove_key_name_field: + type: boolean + replace_invalid_sequence: + type: boolean + reserve_data: + type: boolean + reserve_time: + type: boolean + type: object + prometheus: + properties: + labels: + additionalProperties: + type: string + type: object + metrics: + items: + properties: + buckets: + type: string + desc: + type: string + key: + type: string + labels: + additionalProperties: + type: string + type: object + name: + type: string + type: + type: string + required: + - desc + - name + - type + type: object + type: array + type: object + record_modifier: + properties: + char_encoding: + type: string + prepare_value: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + replaces: + items: + properties: + expression: + type: string + key: + type: string + replace: + type: string + required: + - expression + - key + - replace + type: object + type: array + whitelist_keys: + type: string + type: object + record_transformer: + properties: + auto_typecast: + type: boolean + enable_ruby: + type: boolean + keep_keys: + type: string + records: + items: + additionalProperties: + type: string + type: object + type: array + remove_keys: + type: string + renew_record: + type: boolean + renew_time_key: + type: string + type: object + stdout: + properties: + output_type: + type: string + type: object + sumologic: + properties: + collector_key_name: + type: string + collector_value: + type: string + exclude_container_regex: + type: string + exclude_facility_regex: + type: string + exclude_host_regex: + type: string + exclude_namespace_regex: + type: string + exclude_pod_regex: + type: string + exclude_priority_regex: + type: string + exclude_unit_regex: + type: string + log_format: + type: string + source_category: + type: string + source_category_key_name: + type: string + source_category_prefix: + type: string + source_category_replace_dash: + type: string + source_host: + type: string + source_host_key_name: + type: string + source_name: + type: string + source_name_key_name: + type: string + tracing_annotation_prefix: + type: string + tracing_container_name: + type: string + tracing_format: + type: boolean + tracing_host: + type: string + tracing_label_prefix: + type: string + tracing_namespace: + type: string + tracing_pod: + type: string + tracing_pod_id: + type: string + type: object + tag_normaliser: + properties: + format: + type: string + type: object + throttle: + properties: + group_bucket_limit: + type: integer + group_bucket_period_s: + type: integer + group_drop_logs: + type: boolean + group_key: + type: string + group_reset_rate_s: + type: integer + group_warning_delay_s: + type: integer + type: object + type: object + type: array + loggingRef: + type: string + watchNamespaces: + items: + type: string + type: array + required: + - controlNamespace + type: object + status: + properties: + configCheckResults: + additionalProperties: + type: boolean + type: object + type: object + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_outputs.yaml b/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_outputs.yaml new file mode 100755 index 000000000..b04c809ce --- /dev/null +++ b/charts/rancher-logging/rancher-logging-crd/3.9.002-rc03/templates/logging.banzaicloud.io_outputs.yaml @@ -0,0 +1,4557 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.4 + creationTimestamp: null + name: outputs.logging.banzaicloud.io +spec: + additionalPrinterColumns: + - JSONPath: .status.active + description: Is the output active? + name: Active + type: boolean + - JSONPath: .status.problemsCount + description: Number of problems + name: Problems + type: integer + group: logging.banzaicloud.io + names: + categories: + - logging-all + kind: Output + listKind: OutputList + plural: outputs + singular: output + preserveUnknownFields: false + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + awsElasticsearch: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + endpoint: + properties: + access_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + assume_role_arn: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + assume_role_session_name: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + assume_role_web_identity_token_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + ecs_container_credentials_relative_uri: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + region: + type: string + secret_access_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + sts_credentials_region: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + url: + type: string + type: object + flush_interval: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + include_tag_key: + type: boolean + logstash_format: + type: boolean + tag_key: + type: string + type: object + azurestorage: + properties: + auto_create_container: + type: boolean + azure_container: + type: string + azure_imds_api_version: + type: string + azure_object_key_format: + type: string + azure_storage_access_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + azure_storage_account: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + azure_storage_sas_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + type: string + path: + type: string + required: + - azure_container + - azure_storage_access_key + - azure_storage_account + - azure_storage_sas_token + type: object + cloudwatch: + properties: + auto_create_stream: + type: boolean + aws_instance_profile_credentials_retries: + type: integer + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + aws_sts_role_arn: + type: string + aws_sts_session_name: + type: string + aws_use_sts: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + concurrency: + type: integer + endpoint: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + http_proxy: + type: string + include_time_key: + type: boolean + json_handler: + type: string + localtime: + type: boolean + log_group_aws_tags: + type: string + log_group_aws_tags_key: + type: string + log_group_name: + type: string + log_group_name_key: + type: string + log_rejected_request: + type: string + log_stream_name: + type: string + log_stream_name_key: + type: string + max_events_per_batch: + type: integer + max_message_length: + type: integer + message_keys: + type: string + put_log_events_disable_retry_limit: + type: boolean + put_log_events_retry_limit: + type: integer + put_log_events_retry_wait: + type: string + region: + type: string + remove_log_group_aws_tags_key: + type: string + remove_log_group_name_key: + type: string + remove_log_stream_name_key: + type: string + remove_retention_in_days: + type: string + retention_in_days: + type: string + retention_in_days_key: + type: string + use_tag_as_group: + type: boolean + use_tag_as_stream: + type: boolean + required: + - region + type: object + datadog: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + compression_level: + type: string + dd_hostname: + type: string + dd_source: + type: string + dd_sourcecategory: + type: string + dd_tags: + type: string + host: + type: string + include_tag_key: + type: boolean + max_backoff: + type: string + max_retries: + type: string + no_ssl_validation: + type: boolean + port: + type: string + service: + type: string + ssl_port: + type: string + tag_key: + type: string + timestamp_key: + type: string + use_compression: + type: boolean + use_http: + type: boolean + use_json: + type: boolean + use_ssl: + type: boolean + required: + - api_key + type: object + elasticsearch: + properties: + application_name: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + bulk_message_request_threshold: + type: string + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + client_key_pass: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + content_type: + type: string + custom_headers: + type: string + customize_template: + type: string + default_elasticsearch_version: + type: string + deflector_alias: + type: string + enable_ilm: + type: boolean + exception_backup: + type: boolean + fail_on_putting_template_retry_exceed: + type: boolean + flatten_hashes: + type: boolean + flatten_hashes_separator: + type: string + host: + type: string + hosts: + type: string + http_backend: + type: string + id_key: + type: string + ignore_exceptions: + type: string + ilm_policy: + type: string + ilm_policy_id: + type: string + ilm_policy_overwrite: + type: boolean + include_index_in_url: + type: boolean + include_tag_key: + type: boolean + include_timestamp: + type: boolean + index_date_pattern: + type: string + index_name: + type: string + index_prefix: + type: string + log_es_400_reason: + type: boolean + logstash_dateformat: + type: string + logstash_format: + type: boolean + logstash_prefix: + type: string + logstash_prefix_separator: + type: string + max_retry_get_es_version: + type: string + max_retry_putting_template: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + path: + type: string + pipeline: + type: string + port: + type: integer + prefer_oj_serializer: + type: boolean + reconnect_on_error: + type: boolean + reload_after: + type: string + reload_connections: + type: boolean + reload_on_failure: + type: boolean + remove_keys_on_update: + type: string + remove_keys_on_update_key: + type: string + request_timeout: + type: string + resurrect_after: + type: string + retry_tag: + type: string + rollover_index: + type: boolean + routing_key: + type: string + scheme: + type: string + sniffer_class_name: + type: string + ssl_max_version: + type: string + ssl_min_version: + type: string + ssl_verify: + type: boolean + ssl_version: + type: string + suppress_doc_wrap: + type: boolean + suppress_type_name: + type: boolean + tag_key: + type: string + target_index_key: + type: string + target_type_key: + type: string + template_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + template_name: + type: string + template_overwrite: + type: boolean + templates: + type: string + time_key: + type: string + time_key_format: + type: string + time_parse_error_tag: + type: string + time_precision: + type: string + type_name: + type: string + unrecoverable_error_types: + type: string + user: + type: string + utc_index: + type: boolean + validate_client_version: + type: boolean + verify_es_version_at_startup: + type: boolean + with_transporter_log: + type: boolean + write_operation: + type: string + type: object + file: + properties: + add_path_suffix: + type: boolean + append: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + path: + type: string + path_suffix: + type: string + symlink_path: + type: boolean + required: + - path + type: object + forward: + properties: + ack_response_timeout: + type: integer + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + connect_timeout: + type: integer + dns_round_robin: + type: boolean + expire_dns_cache: + type: integer + hard_timeout: + type: integer + heartbeat_interval: + type: integer + heartbeat_type: + type: string + ignore_network_errors_at_startup: + type: boolean + keepalive: + type: boolean + keepalive_timeout: + type: integer + phi_failure_detector: + type: boolean + phi_threshold: + type: integer + recover_wait: + type: integer + require_ack_response: + type: boolean + security: + properties: + allow_anonymous_source: + type: boolean + self_hostname: + type: string + shared_key: + type: string + user_auth: + type: boolean + required: + - self_hostname + - shared_key + type: object + send_timeout: + type: integer + servers: + items: + properties: + host: + type: string + name: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + port: + type: integer + shared_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + standby: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + weight: + type: integer + required: + - host + type: object + type: array + tls_allow_self_signed_cert: + type: boolean + tls_cert_logical_store_name: + type: string + tls_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + tls_cert_thumbprint: + type: string + tls_cert_use_enterprise_store: + type: boolean + tls_ciphers: + type: string + tls_client_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + tls_client_private_key_passphrase: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + tls_client_private_key_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + tls_insecure_mode: + type: boolean + tls_verify_hostname: + type: boolean + tls_version: + type: string + verify_connection_at_startup: + type: boolean + required: + - servers + type: object + gcs: + properties: + acl: + type: string + auto_create_bucket: + type: boolean + bucket: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + client_retries: + type: integer + client_timeout: + type: integer + credentials_json: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + encryption_key: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + hex_random_length: + type: integer + keyfile: + type: string + object_key_format: + type: string + object_metadata: + items: + properties: + key: + type: string + value: + type: string + required: + - key + - value + type: object + type: array + overwrite: + type: boolean + path: + type: string + project: + type: string + storage_class: + type: string + store_as: + type: string + transcoding: + type: boolean + required: + - bucket + - project + type: object + http: + properties: + auth: + properties: + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + required: + - password + - username + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + content_type: + type: string + endpoint: + type: string + error_response_as_unrecoverable: + type: boolean + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + headers: + additionalProperties: + type: string + type: object + http_method: + type: string + json_array: + type: boolean + open_timeout: + type: integer + proxy: + type: string + read_timeout: + type: integer + retryable_response_codes: + items: + type: integer + type: array + ssl_timeout: + type: integer + tls_ca_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + tls_ciphers: + type: string + tls_client_cert_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + tls_private_key_passphrase: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + tls_private_key_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + tls_verify_mode: + type: string + tls_version: + type: string + required: + - endpoint + type: object + kafka: + properties: + ack_timeout: + type: integer + brokers: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + compression_codec: + type: string + default_message_key: + type: string + default_partition_key: + type: string + default_topic: + type: string + exclude_partion_key: + type: boolean + exclude_topic_key: + type: boolean + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + get_kafka_client_log: + type: boolean + headers: + additionalProperties: + type: string + type: object + headers_from_record: + additionalProperties: + type: string + type: object + idempotent: + type: boolean + max_send_retries: + type: integer + message_key_key: + type: string + partition_key: + type: string + partition_key_key: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + required_acks: + type: integer + sasl_over_ssl: + type: boolean + scram_mechanism: + type: string + ssl_ca_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + ssl_ca_certs_from_system: + type: boolean + ssl_client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + ssl_client_cert_chain: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + ssl_client_cert_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + ssl_verify_hostname: + type: boolean + topic_key: + type: string + use_default_for_unknown_topic: + type: boolean + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + required: + - brokers + - format + type: object + kinesisStream: + properties: + assume_role_credentials: + properties: + duration_seconds: + type: string + external_id: + type: string + policy: + type: string + role_arn: + type: string + role_session_name: + type: string + required: + - role_arn + - role_session_name + type: object + aws_iam_retries: + type: integer + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + aws_ses_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + batch_request_max_count: + type: integer + batch_request_max_size: + type: integer + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + partition_key: + type: string + process_credentials: + properties: + process: + type: string + required: + - process + type: object + region: + type: string + reset_backoff_if_success: + type: boolean + retries_on_batch_request: + type: integer + stream_name: + type: string + required: + - stream_name + type: object + logdna: + properties: + api_key: + type: string + app: + type: string + buffer_chunk_limit: + type: string + hostname: + type: string + required: + - api_key + - hostname + type: object + loggingRef: + type: string + logz: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + endpoint: + properties: + port: + type: integer + token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + url: + type: string + type: object + gzip: + type: boolean + http_idle_timeout: + type: integer + output_include_tags: + type: boolean + output_include_time: + type: boolean + retry_count: + type: integer + retry_sleep: + type: integer + required: + - endpoint + type: object + loki: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + configure_kubernetes_labels: + type: boolean + drop_single_key: + type: boolean + extra_labels: + additionalProperties: + type: string + type: object + extract_kubernetes_labels: + type: boolean + insecure_tls: + type: boolean + key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + labels: + additionalProperties: + type: string + type: object + line_format: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + remove_keys: + items: + type: string + type: array + tenant: + type: string + url: + type: string + username: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + type: object + newrelic: + properties: + api_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + base_uri: + type: string + license_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + type: object + nullout: + type: object + oss: + properties: + aaccess_key_secret: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + access_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + auto_create_bucket: + type: boolean + bucket: + type: string + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + check_bucket: + type: boolean + check_object: + type: boolean + download_crc_enable: + type: boolean + endpoint: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + hex_random_length: + type: integer + index_format: + type: string + key_format: + type: string + open_timeout: + type: integer + oss_sdk_log_dir: + type: string + overwrite: + type: boolean + path: + type: string + read_timeout: + type: integer + store_as: + type: string + upload_crc_enable: + type: boolean + warn_for_delay: + type: string + required: + - aaccess_key_secret + - access_key_id + - bucket + - endpoint + type: object + redis: + properties: + allow_duplicate_key: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + db_number: + type: integer + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + host: + type: string + insert_key_prefix: + type: string + password: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + port: + type: integer + strftime_format: + type: string + ttl: + type: integer + type: object + s3: + properties: + acl: + type: string + assume_role_credentials: + properties: + duration_seconds: + type: string + external_id: + type: string + policy: + type: string + role_arn: + type: string + role_session_name: + type: string + required: + - role_arn + - role_session_name + type: object + auto_create_bucket: + type: string + aws_iam_retries: + type: string + aws_key_id: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + aws_sec_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + check_apikey_on_start: + type: string + check_bucket: + type: string + check_object: + type: string + clustername: + type: string + compute_checksums: + type: string + enable_transfer_acceleration: + type: string + force_path_style: + type: string + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + grant_full_control: + type: string + grant_read: + type: string + grant_read_acp: + type: string + grant_write_acp: + type: string + hex_random_length: + type: string + index_format: + type: string + instance_profile_credentials: + properties: + http_open_timeout: + type: string + http_read_timeout: + type: string + ip_address: + type: string + port: + type: string + retries: + type: string + type: object + oneeye_format: + type: boolean + overwrite: + type: string + path: + type: string + proxy_uri: + type: string + s3_bucket: + type: string + s3_endpoint: + type: string + s3_metadata: + type: string + s3_object_key_format: + type: string + s3_region: + type: string + shared_credentials: + properties: + path: + type: string + profile_name: + type: string + type: object + signature_version: + type: string + sse_customer_algorithm: + type: string + sse_customer_key: + type: string + sse_customer_key_md5: + type: string + ssekms_key_id: + type: string + ssl_verify_peer: + type: string + storage_class: + type: string + store_as: + type: string + use_bundled_cert: + type: string + use_server_side_encryption: + type: string + warn_for_delay: + type: string + required: + - s3_bucket + type: object + splunkHec: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + ca_file: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + ca_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + client_cert: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + client_key: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + coerce_to_utf8: + type: boolean + data_type: + type: string + fields: + additionalProperties: + type: string + type: object + format: + properties: + add_newline: + type: boolean + message_key: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + hec_host: + type: string + hec_port: + type: integer + hec_token: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + host: + type: string + host_key: + type: string + idle_timeout: + type: integer + index: + type: string + index_key: + type: string + insecure_ssl: + type: boolean + keep_keys: + type: boolean + metric_name_key: + type: string + metric_value_key: + type: string + metrics_from_event: + type: boolean + non_utf8_replacement_string: + type: string + open_timeout: + type: integer + protocol: + type: string + read_timeout: + type: integer + source: + type: string + source_key: + type: string + sourcetype: + type: string + sourcetype_key: + type: string + ssl_ciphers: + type: string + required: + - hec_host + - hec_token + type: object + sumologic: + properties: + add_timestamp: + type: boolean + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + compress: + type: boolean + compress_encoding: + type: string + custom_dimensions: + type: string + custom_fields: + items: + type: string + type: array + data_type: + type: string + delimiter: + type: string + disable_cookies: + type: boolean + endpoint: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + log_format: + type: string + log_key: + type: string + metric_data_format: + type: string + open_timeout: + type: integer + proxy_uri: + type: string + source_category: + type: string + source_host: + type: string + source_name: + type: string + source_name_key: + type: string + sumo_client: + type: string + timestamp_key: + type: string + verify_ssl: + type: boolean + required: + - endpoint + - source_name + type: object + syslog: + properties: + buffer: + properties: + chunk_full_threshold: + type: string + chunk_limit_records: + type: integer + chunk_limit_size: + type: string + compress: + type: string + delayed_commit_timeout: + type: string + disable_chunk_backup: + type: boolean + flush_at_shutdown: + type: boolean + flush_interval: + type: string + flush_mode: + type: string + flush_thread_burst_interval: + type: string + flush_thread_count: + type: integer + flush_thread_interval: + type: string + overflow_action: + type: string + path: + type: string + queue_limit_length: + type: integer + queued_chunks_limit_size: + type: integer + retry_exponential_backoff_base: + type: string + retry_forever: + type: boolean + retry_max_interval: + type: string + retry_max_times: + type: integer + retry_randomize: + type: boolean + retry_secondary_threshold: + type: string + retry_timeout: + type: string + retry_type: + type: string + retry_wait: + type: string + tags: + type: string + timekey: + type: string + timekey_use_utc: + type: boolean + timekey_wait: + type: string + timekey_zone: + type: string + total_limit_size: + type: string + type: + type: string + type: object + format: + properties: + app_name_field: + type: string + hostname_field: + type: string + log_field: + type: string + message_id_field: + type: string + proc_id_field: + type: string + rfc6587_message_size: + type: boolean + structured_data_field: + type: string + type: + enum: + - out_file + - json + - ltsv + - csv + - msgpack + - hash + - single_value + type: string + type: object + host: + type: string + insecure: + type: boolean + port: + type: integer + transport: + type: string + trusted_ca_path: + properties: + mountFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + value: + type: string + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + type: object + type: object + required: + - host + type: object + type: object + status: + properties: + active: + type: boolean + problems: + items: + type: string + type: array + problemsCount: + type: integer + type: object + type: object + version: v1beta1 + versions: + - name: v1beta1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/.helmignore b/charts/rancher-logging/rancher-logging/3.9.002-rc03/.helmignore new file mode 100755 index 000000000..50af03172 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/Chart.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/Chart.yaml new file mode 100755 index 000000000..8466cd529 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/Chart.yaml @@ -0,0 +1,20 @@ +annotations: + catalog.cattle.io/auto-install: rancher-logging-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: Logging + catalog.cattle.io/namespace: cattle-logging-system + catalog.cattle.io/os: linux + catalog.cattle.io/provides-gvr: logging.banzaicloud.io.clusterflow/v1beta1 + catalog.cattle.io/release-name: rancher-logging + catalog.cattle.io/ui-component: logging +apiVersion: v1 +appVersion: 3.9.0 +description: Collects and filter logs using highly configurable CRDs. Powered by Banzai + Cloud Logging Operator. +icon: https://charts.rancher.io/assets/logos/logging.svg +keywords: +- logging +- monitoring +- security +name: rancher-logging +version: 3.9.002-rc03 diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/README.md b/charts/rancher-logging/rancher-logging/3.9.002-rc03/README.md new file mode 100755 index 000000000..ca7938e2a --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/README.md @@ -0,0 +1,130 @@ + +# Logging operator Chart + +[Logging operator](https://github.com/banzaicloud/logging-operator) Managed centralized logging component fluentd and fluent-bit instance on cluster. + +## tl;dr: + +```bash +$ helm repo add banzaicloud-stable https://kubernetes-charts.banzaicloud.com +$ helm repo update +$ helm install banzaicloud-stable/logging-operator +``` + +## Introduction + +This chart bootstraps a [Logging Operator](https://github.com/banzaicloud/logging-operator) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Kubernetes 1.8+ with Beta APIs enabled + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```bash +$ helm install --name my-release banzaicloud-stable/logging-operator +``` + +### CRDs +Use `createCustomResource=false` with Helm v3 to avoid trying to create CRDs from the `crds` folder and from templates at the same time. + +The command deploys **Logging operator** on the Kubernetes cluster with the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. + +## Uninstalling the Chart + +To uninstall/delete the `my-release` deployment: + +```bash +$ helm delete my-release +``` + +The command removes all Kubernetes components associated with the chart and deletes the release. + +## Configuration + +The following tables lists the configurable parameters of the logging-operator chart and their default values. + +| Parameter | Description | Default | +| --------------------------------------------------- | ------------------------------------------------------ | ------------------------------ | +| `image.repository` | Container image repository | `ghcr.io/banzaicloud/logging-operator` | +| `image.tag` | Container image tag | `3.9.0` | +| `image.pullPolicy` | Container pull policy | `IfNotPresent` | +| `nameOverride` | Override name of app | `` | +| `fullnameOverride` | Override full name of app | `` | +| `namespaceOverride` | Override namespace of app | `` | +| `watchNamespace` | Namespace to watch for LoggingOperator CRD | `` | +| `rbac.enabled` | Create rbac service account and roles | `true` | +| `rbac.psp.enabled` | Must be used with `rbac.enabled` true. If true, creates & uses RBAC resources required in the cluster with [Pod Security Policies](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) enabled. | `false` | +| `priorityClassName` | Operator priorityClassName | `{}` | +| `affinity` | Node Affinity | `{}` | +| `resources` | CPU/Memory resource requests/limits | `{}` | +| `tolerations` | Node Tolerations | `[]` | +| `nodeSelector` | Define which Nodes the Pods are scheduled on. | `{}` | +| `annotations` | Define annotations for logging-operator pods | `{}` | +| `podSecurityContext` | Pod SecurityContext for Logging operator. [More info](https://kubernetes.io/docs/concepts/policy/security-context/) | `{"runAsNonRoot": true, "runAsUser": 1000, "fsGroup": 2000}` | +| `securityContext` | Container SecurityContext for Logging operator. [More info](https://kubernetes.io/docs/concepts/policy/security-context/) | `{"allowPrivilegeEscalation": false, "readOnlyRootFilesystem": true}` | +| `createCustomResource` | Create CRDs. | `true` | +| `monitoring.serviceMonitor.enabled` | Create Prometheus Operator servicemonitor. | `false` | +| `global.seLinux.enabled` | Add seLinuxOptions to Logging resources, requires the [rke2-selinux RPM](https://github.com/rancher/rke2-selinux/releases) | `false` | + +Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example: + +```bash +$ helm install --name my-release -f values.yaml banzaicloud-stable/logging-operator +``` + +> **Tip**: You can use the default [values.yaml](values.yaml) + +## Installing Fluentd and Fluent-bit via logging + +The previous chart does **not** install `logging` resource to deploy Fluentd and Fluent-bit on cluster. To install them please use the [Logging Operator Logging](https://github.com/banzaicloud/logging-operator/tree/master/charts/logging-operator-logging) chart. + +## tl;dr: + +```bash +$ helm repo add banzaicloud-stable https://kubernetes-charts.banzaicloud.com +$ helm repo update +$ helm install banzaicloud-stable/logging-operator-logging +``` + +## Configuration + +The following tables lists the configurable parameters of the logging-operator-logging chart and their default values. +## tl;dr: + +```bash +$ helm repo add banzaicloud-stable https://kubernetes-charts.banzaicloud.com +$ helm repo update +$ helm install banzaicloud-stable/logging-operator-logging +``` + +## Configuration + +The following tables lists the configurable parameters of the logging-operator-logging chart and their default values. + +| Parameter | Description | Default | +| --------------------------------------------------- | ------------------------------------------------------ | ------------------------------ | +| `tls.enabled` | Enabled TLS communication between components | true | +| `tls.fluentdSecretName` | Specified secret name, which contain tls certs | This will overwrite automatic Helm certificate generation. | +| `tls.fluentbitSecretName` | Specified secret name, which contain tls certs | This will overwrite automatic Helm certificate generation. | +| `tls.sharedKey` | Shared key between nodes (fluentd-fluentbit) | [autogenerated] | +| `fluentbit.enabled` | Install fluent-bit | true | +| `fluentbit.namespace` | Specified fluentbit installation namespace | same as operator namespace | +| `fluentbit.image.tag` | Fluentbit container image tag | `1.6.10` | +| `fluentbit.image.repository` | Fluentbit container image repository | `fluent/fluent-bit` | +| `fluentbit.image.pullPolicy` | Fluentbit container pull policy | `IfNotPresent` | +| `fluentd.enabled` | Install fluentd | true | +| `fluentd.image.tag` | Fluentd container image tag | `v1.11.5-alpine-9` | +| `fluentd.image.repository` | Fluentd container image repository | `ghcr.io/banzaicloud/fluentd` | +| `fluentd.image.pullPolicy` | Fluentd container pull policy | `IfNotPresent` | +| `fluentd.volumeModImage.tag` | Fluentd volumeModImage container image tag | `latest` | +| `fluentd.volumeModImage.repository` | Fluentd volumeModImage container image repository | `busybox` | +| `fluentd.volumeModImage.pullPolicy` | Fluentd volumeModImage container pull policy | `IfNotPresent` | +| `fluentd.configReloaderImage.tag` | Fluentd configReloaderImage container image tag | `v0.2.2` | +| `fluentd.configReloaderImage.repository` | Fluentd configReloaderImage container image repository | `jimmidyson/configmap-reload` | +| `fluentd.configReloaderImage.pullPolicy` | Fluentd configReloaderImage container pull policy | `IfNotPresent` | +| `fluentd.fluentdPvcSpec.accessModes` | Fluentd persistence volume access modes | `[ReadWriteOnce]` | +| `fluentd.fluentdPvcSpec.resources.requests.storage` | Fluentd persistence volume size | `21Gi` | +| `fluentd.fluentdPvcSpec.storageClassName` | Fluentd persistence volume storageclass | `"""` | diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/app-readme.md b/charts/rancher-logging/rancher-logging/3.9.002-rc03/app-readme.md new file mode 100755 index 000000000..2de4ab4c5 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/app-readme.md @@ -0,0 +1,22 @@ +# Rancher Logging + +This chart is based off of the upstream [Banzai Logging Operator](https://banzaicloud.com/docs/one-eye/logging-operator/) chart. The chart deploys a logging operator and CRDs, which allows users to configure complex logging pipelines with a few simple custom resources. There are two levels of logging, which allow you to collect all logs in a cluster or from a single namespace. + +For more information on how to use the feature, refer to our [docs](https://rancher.com/docs/rancher/v2.x/en/logging/v2.5/). + +## Namespace-level logging + +To collect logs from a single namespace, users create flows and these flows are connected to outputs or cluster outputs. + +## Cluster-level logging + +To collect logs from an entire cluster, users create cluster flows and cluster outputs. + +## CRDs + +- [Cluster Flow](https://banzaicloud.com/docs/one-eye/logging-operator/crds/v1beta1/clusterflow_types/) - A cluster flow is a CRD (`ClusterFlow`) that defines what logs to collect from the entire cluster. The cluster flow must be deployed in the same namespace as the logging operator. +- [Cluster Output](https://banzaicloud.com/docs/one-eye/logging-operator/crds/v1beta1/clusteroutput_types/) - A cluster output is a CRD (`ClusterOutput`) that defines how to connect to logging providers so they can start collecting logs. The cluster output must be deployed in the same namespace as the logging operator. The convenience of using a cluster output is that either a cluster flow or flow can send logs to those providers without needing to define specific outputs in each namespace for each flow. +- [Flow](https://banzaicloud.com/docs/one-eye/logging-operator/crds/v1beta1/flow_types/) - A flow is a CRD (`Flow`) that defines what logs to collect from the namespace that it is deployed in. +- [Output](https://banzaicloud.com/docs/one-eye/logging-operator/crds/v1beta1/output_types/) - An output is a CRD (`Output`) that defines how to connect to logging providers so logs can be sent to the provider. + +For more information on how to configure the Helm chart, refer to the Helm README. diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/NOTES.txt b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/NOTES.txt new file mode 100755 index 000000000..e69de29bb diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/_helpers.tpl b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/_helpers.tpl new file mode 100755 index 000000000..b2b289443 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/_helpers.tpl @@ -0,0 +1,66 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "logging-operator.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "logging-operator.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Provides the namespace the chart will be installed in using the builtin .Release.Namespace, +or, if provided, a manually overwritten namespace value. +*/}} +{{- define "logging-operator.namespace" -}} +{{- if .Values.namespaceOverride -}} +{{ .Values.namespaceOverride -}} +{{- else -}} +{{ .Release.Namespace }} +{{- end -}} +{{- end -}} + + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "logging-operator.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "logging-operator.labels" -}} +app.kubernetes.io/name: {{ include "logging-operator.name" . }} +helm.sh/chart: {{ include "logging-operator.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/clusterrole.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/clusterrole.yaml new file mode 100755 index 000000000..709eedb91 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/clusterrole.yaml @@ -0,0 +1,167 @@ +{{- if .Values.rbac.enabled }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: {{ template "logging-operator.fullname" . }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - endpoints + - namespaces + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + - pods + - serviceaccounts + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + - events.k8s.io + resources: + - events + verbs: + - create + - get + - list + - watch +- apiGroups: + - apps + resources: + - daemonsets + - replicasets + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + - extensions + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - extensions + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - extensions + - policy + resources: + - podsecuritypolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - use + - watch +- apiGroups: + - logging.banzaicloud.io + resources: + - clusterflows + - clusteroutputs + - flows + - loggings + - outputs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - logging.banzaicloud.io + resources: + - clusterflows/status + - clusteroutputs/status + - flows/status + - loggings/status + - outputs/status + verbs: + - get + - patch + - update +- apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + - clusterroles + - rolebindings + - roles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +{{- end }} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/clusterrolebinding.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..89d17d094 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if .Values.rbac.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "logging-operator.fullname" . }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +subjects: + - kind: ServiceAccount + name: {{ template "logging-operator.fullname" . }} + namespace: {{ include "logging-operator.namespace" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "logging-operator.fullname" . }} + + {{- end }} \ No newline at end of file diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/crds.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/crds.yaml new file mode 100755 index 000000000..f573652d0 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/crds.yaml @@ -0,0 +1,6 @@ +{{- if .Values.createCustomResource -}} +{{- range $path, $bytes := .Files.Glob "crds/*.yaml" }} +{{ $.Files.Get $path }} +--- +{{- end }} +{{- end }} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/deployment.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/deployment.yaml new file mode 100755 index 000000000..da93d4c29 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/deployment.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "logging-operator.fullname" . }} + namespace: {{ include "logging-operator.namespace" . }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "logging-operator.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "logging-operator.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + ports: + - name: http + containerPort: {{ .Values.http.port }} + + {{- if .Values.securityContext }} + securityContext: {{ toYaml .Values.securityContext | nindent 12 }} + {{- end }} + {{- if .Values.podSecurityContext }} + securityContext: {{ toYaml .Values.podSecurityContext | nindent 8 }} + {{- end }} + + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.rbac.enabled }} + serviceAccountName: {{ include "logging-operator.fullname" . }} + {{- end }} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/aks/logging.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/aks/logging.yaml new file mode 100755 index 000000000..916f93b41 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/aks/logging.yaml @@ -0,0 +1,58 @@ +{{- if .Values.additionalLoggingSources.aks.enabled }} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Logging +metadata: + name: {{ .Release.Name }}-aks + namespace: {{ .Release.Namespace }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +spec: + controlNamespace: {{ .Release.Namespace }} + fluentbit: + image: + repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentbit.repository }} + tag: {{ .Values.images.fluentbit.tag }} + inputTail: + Tag: "aks" + Path: "/var/log/azure/kubelet-status.log" + {{- if .Values.global.psp.enabled }} + security: + podSecurityPolicyCreate: true + roleBasedAccessControlCreate: true + {{- end }} + {{- $total_tolerations := concat (.Values.tolerations) (.Values.fluentbit.tolerations) }} + {{- with $total_tolerations }} + tolerations: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentbit.resources }} + resources: {{- toYaml . | nindent 6 }} + {{- end }} + fluentd: + image: + repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentd.repository }} + tag: {{ .Values.images.fluentd.tag }} + configReloaderImage: + repository: {{ template "system_default_registry" . }}{{ .Values.images.config_reloader.repository }} + tag: {{ .Values.images.config_reloader.tag }} + disablePvc: {{ .Values.disablePvc }} + {{- if .Values.global.psp.enabled }} + security: + podSecurityPolicyCreate: true + roleBasedAccessControlCreate: true + {{- end }} + {{- with .Values.tolerations }} + tolerations: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentd.resources }} + resources: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentd.livenessProbe }} + livenessProbe: {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/eks/logging.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/eks/logging.yaml new file mode 100755 index 000000000..da4af2d9e --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/eks/logging.yaml @@ -0,0 +1,59 @@ +{{- if .Values.additionalLoggingSources.eks.enabled }} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Logging +metadata: + name: {{ .Release.Name }}-eks + namespace: {{ .Release.Namespace }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +spec: + controlNamespace: {{ .Release.Namespace }} + fluentbit: + image: + repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentbit.repository }} + tag: {{ .Values.images.fluentbit.tag }} + inputTail: + Tag: "eks" + Path: "/var/log/messages" + Parser: "syslog" + {{- if .Values.global.psp.enabled }} + security: + podSecurityPolicyCreate: true + roleBasedAccessControlCreate: true + {{- end }} + {{- $total_tolerations := concat (.Values.tolerations) (.Values.fluentbit.tolerations) }} + {{- with $total_tolerations }} + tolerations: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentbit.resources }} + resources: {{- toYaml . | nindent 6 }} + {{- end }} + fluentd: + image: + repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentd.repository }} + tag: {{ .Values.images.fluentd.tag }} + configReloaderImage: + repository: {{ template "system_default_registry" . }}{{ .Values.images.config_reloader.repository }} + tag: {{ .Values.images.config_reloader.tag }} + disablePvc: {{ .Values.disablePvc }} + {{- if .Values.global.psp.enabled }} + security: + podSecurityPolicyCreate: true + roleBasedAccessControlCreate: true + {{- end }} + {{- with .Values.tolerations }} + tolerations: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentd.resources }} + resources: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentd.livenessProbe }} + livenessProbe: {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/gke/logging.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/gke/logging.yaml new file mode 100755 index 000000000..3823127b2 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/gke/logging.yaml @@ -0,0 +1,58 @@ +{{- if .Values.additionalLoggingSources.gke.enabled }} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Logging +metadata: + name: {{ .Release.Name }}-gke + namespace: {{ .Release.Namespace }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +spec: + controlNamespace: {{ .Release.Namespace }} + fluentbit: + image: + repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentbit.repository }} + tag: {{ .Values.images.fluentbit.tag }} + inputTail: + Tag: "gke" + Path: "/var/log/kube-proxy.log" + {{- if .Values.global.psp.enabled }} + security: + podSecurityPolicyCreate: true + roleBasedAccessControlCreate: true + {{- end }} + {{- $total_tolerations := concat (.Values.tolerations) (.Values.fluentbit.tolerations) }} + {{- with $total_tolerations }} + tolerations: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentbit.resources }} + resources: {{- toYaml . | nindent 6 }} + {{- end }} + fluentd: + image: + repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentd.repository }} + tag: {{ .Values.images.fluentd.tag }} + configReloaderImage: + repository: {{ template "system_default_registry" . }}{{ .Values.images.config_reloader.repository }} + tag: {{ .Values.images.config_reloader.tag }} + disablePvc: {{ .Values.disablePvc }} + {{- if .Values.global.psp.enabled }} + security: + podSecurityPolicyCreate: true + roleBasedAccessControlCreate: true + {{- end }} + {{- with .Values.tolerations }} + tolerations: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentd.resources }} + resources: {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentd.livenessProbe }} + livenessProbe: {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/k3s/logging-k3s-openrc.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/k3s/logging-k3s-openrc.yaml new file mode 100755 index 000000000..cf5e988f3 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/k3s/logging-k3s-openrc.yaml @@ -0,0 +1,68 @@ +{{- if and .Values.additionalLoggingSources.k3s.enabled (eq .Values.additionalLoggingSources.k3s.container_engine "openrc")}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Logging +metadata: + name: {{ .Release.Name }}-k3s + namespace: {{ .Release.Namespace }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +spec: + controlNamespace: {{ .Release.Namespace }} + fluentbit: + image: + repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentbit.repository }} + tag: {{ .Values.images.fluentbit.tag }} + inputTail: + Tag: "k3s" + Path: "/var/log/k3s.log" + extraVolumeMounts: + - source: "/var/log/" + destination: "/var/log" + readOnly: true + {{- if .Values.global.psp.enabled }} + security: + podSecurityPolicyCreate: true + roleBasedAccessControlCreate: true + {{- end }} + {{- $total_tolerations := concat (.Values.tolerations) (.Values.fluentbit.tolerations) }} + {{- with $total_tolerations }} + tolerations: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentbit.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + fluentd: + image: + repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentd.repository }} + tag: {{ .Values.images.fluentd.tag }} + configReloaderImage: + repository: {{ template "system_default_registry" . }}{{ .Values.images.config_reloader.repository }} + tag: {{ .Values.images.config_reloader.tag }} + disablePvc: {{ .Values.disablePvc }} + {{- if .Values.global.psp.enabled }} + security: + podSecurityPolicyCreate: true + roleBasedAccessControlCreate: true + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentd.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentd.livenessProbe }} + livenessProbe: {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/k3s/logging-k3s-systemd.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/k3s/logging-k3s-systemd.yaml new file mode 100755 index 000000000..c4b3db0e7 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/k3s/logging-k3s-systemd.yaml @@ -0,0 +1,68 @@ +{{- if and .Values.additionalLoggingSources.k3s.enabled (eq .Values.additionalLoggingSources.k3s.container_engine "systemd")}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Logging +metadata: + name: {{ .Release.Name }}-k3s + namespace: {{ .Release.Namespace }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +spec: + controlNamespace: {{ .Release.Namespace }} + fluentbit: + image: + repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentbit.repository }} + tag: {{ .Values.images.fluentbit.tag }} + inputTail: + Tag: "k3s" + Path: "/var/log/syslog" + extraVolumeMounts: + - source: "/var/log/" + destination: "/var/log" + readOnly: true + {{- if .Values.global.psp.enabled }} + security: + podSecurityPolicyCreate: true + roleBasedAccessControlCreate: true + {{- end }} + {{- $total_tolerations := concat (.Values.tolerations) (.Values.fluentbit.tolerations) }} + {{- with $total_tolerations }} + tolerations: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentbit.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + fluentd: + image: + repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentd.repository }} + tag: {{ .Values.images.fluentd.tag }} + configReloaderImage: + repository: {{ template "system_default_registry" . }}{{ .Values.images.config_reloader.repository }} + tag: {{ .Values.images.config_reloader.tag }} + disablePvc: {{ .Values.disablePvc }} + {{- if .Values.global.psp.enabled }} + security: + podSecurityPolicyCreate: true + roleBasedAccessControlCreate: true + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentd.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentd.livenessProbe }} + livenessProbe: {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke/configmap.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke/configmap.yaml new file mode 100755 index 000000000..ab91d93e2 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke/configmap.yaml @@ -0,0 +1,29 @@ +{{- if .Values.additionalLoggingSources.rke.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-rke + labels: +{{ include "logging-operator.labels" . | indent 4 }} +data: + fluent-bit.conf: | + [SERVICE] + Log_Level {{ .Values.additionalLoggingSources.rke.fluentbit.log_level }} + Parsers_File parsers.conf + + [INPUT] + Tag rke + Name tail + Path_Key filename + Parser docker + DB /tail-db/tail-containers-state.db + Mem_Buf_Limit {{ .Values.additionalLoggingSources.rke.fluentbit.mem_buffer_limit }} + Path /var/lib/rancher/rke/log/*.log + + [OUTPUT] + Name forward + Match * + Host {{ .Release.Name }}-fluentd.{{ .Release.Namespace }}.svc + Port 24240 + Retry_Limit False +{{- end }} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke/daemonset.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke/daemonset.yaml new file mode 100755 index 000000000..840b3e722 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke/daemonset.yaml @@ -0,0 +1,124 @@ +{{- if .Values.additionalLoggingSources.rke.enabled }} +{{- $containers := printf "%s/containers/" (default "/var/lib/docker" .Values.global.dockerRootDirectory) }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: "{{ .Release.Name }}-rke-aggregator" + namespace: "{{ .Release.Namespace }}" +spec: + selector: + matchLabels: + name: {{ .Release.Name }}-rke-aggregator + template: + metadata: + name: "{{ .Release.Name }}-rke-aggregator" + namespace: "{{ .Release.Namespace }}" + labels: + name: {{ .Release.Name }}-rke-aggregator + spec: + containers: + - name: fluentbit + image: "{{ template "system_default_registry" . }}{{ .Values.images.fluentbit.repository }}:{{ .Values.images.fluentbit.tag }}" + volumeMounts: + - mountPath: /var/lib/rancher/rke/log/ + name: indir + - mountPath: {{ $containers }} + name: containers + - mountPath: /tail-db + name: positiondb + - mountPath: /fluent-bit/etc/fluent-bit.conf + name: config + subPath: fluent-bit.conf + {{- if .Values.global.seLinux.enabled }} + securityContext: + seLinuxOptions: + type: rke_logreader_t + {{- end }} + volumes: + - name: indir + hostPath: + path: /var/lib/rancher/rke/log/ + type: DirectoryOrCreate + - name: containers + hostPath: + path: {{ $containers }} + type: DirectoryOrCreate + - name: positiondb + emptyDir: {} + - name: config + configMap: + name: "{{ .Release.Name }}-rke" + serviceAccountName: "{{ .Release.Name }}-rke-aggregator" + {{- $total_tolerations := concat (.Values.tolerations) (.Values.fluentbit.tolerations) }} + {{- with $total_tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "{{ .Release.Name }}-rke-aggregator" + namespace: "{{ .Release.Namespace }}" +{{- if .Values.global.psp.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: "{{ .Release.Name }}-rke-aggregator" +rules: + - apiGroups: + - policy + resourceNames: + - "{{ .Release.Name }}-rke-aggregator" + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: "{{ .Release.Name }}-rke-aggregator" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: "{{ .Release.Name }}-rke-aggregator" +subjects: + - kind: ServiceAccount + name: "{{ .Release.Name }}-rke-aggregator" +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: "{{ .Release.Name }}-rke-aggregator" + namespace: "{{ .Release.Namespace }}" +spec: + allowPrivilegeEscalation: false + allowedHostPaths: + - pathPrefix: {{ $containers }} + readOnly: false + - pathPrefix: /var/lib/rancher/rke/log/ + readOnly: false + - pathPrefix: /var/lib/rancher/logging/ + readOnly: false + fsGroup: + rule: RunAsAny + readOnlyRootFilesystem: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - configMap + - emptyDir + - secret + - hostPath +{{- end }} +{{- end }} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke2/configmap.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke2/configmap.yaml new file mode 100755 index 000000000..86369d140 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke2/configmap.yaml @@ -0,0 +1,21 @@ +{{- if .Values.additionalLoggingSources.rke2.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-rke2 + labels: +{{ include "logging-operator.labels" . | indent 4 }} +data: + fluent-bit.conf: | + [INPUT] + Name systemd + Tag rke2 + Systemd_Filter _SYSTEMD_UNIT=rke2.service + + [OUTPUT] + Name forward + Match * + Host {{ .Release.Name }}-fluentd.{{ .Release.Namespace }}.svc + Port 24240 + Retry_Limit False +{{- end }} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke2/daemonset.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke2/daemonset.yaml new file mode 100755 index 000000000..580522259 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke2/daemonset.yaml @@ -0,0 +1,104 @@ +{{- if .Values.additionalLoggingSources.rke2.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: "{{ .Release.Name }}-rke2-journald-aggregator" + namespace: "{{ .Release.Namespace }}" +spec: + selector: + matchLabels: + name: {{ .Release.Name }}-rke2-journald-aggregator + template: + metadata: + name: "{{ .Release.Name }}-rke2-journald-aggregator" + namespace: "{{ .Release.Namespace }}" + labels: + name: {{ .Release.Name }}-rke2-journald-aggregator + spec: + containers: + - name: fluentd + image: "{{ template "system_default_registry" . }}{{ .Values.images.fluentbit.repository }}:{{ .Values.images.fluentbit.tag }}" + {{- if .Values.global.seLinux.enabled }} + securityContext: + seLinuxOptions: + type: rke_logreader_t + {{- end }} + volumeMounts: + - mountPath: /fluent-bit/etc/ + name: config + - mountPath: /run/log/journal + name: journal + readOnly: true + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: "{{ .Release.Name }}-rke2-journald-aggregator" + volumes: + - name: config + configMap: + name: "{{ .Release.Name }}-rke2" + - name: journal + hostPath: + path: /run/log/journal +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "{{ .Release.Name }}-rke2-journald-aggregator" + namespace: "{{ .Release.Namespace }}" +{{- if .Values.global.psp.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: "{{ .Release.Name }}-rke2-journald-aggregator" +rules: + - apiGroups: + - policy + resourceNames: + - "{{ .Release.Name }}-rke2-journald-aggregator" + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: "{{ .Release.Name }}-rke2-journald-aggregator" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: "{{ .Release.Name }}-rke2-journald-aggregator" +subjects: + - kind: ServiceAccount + name: "{{ .Release.Name }}-rke2-journald-aggregator" +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: "{{ .Release.Name }}-rke2-journald-aggregator" + namespace: "{{ .Release.Namespace }}" +spec: + allowPrivilegeEscalation: false + fsGroup: + rule: RunAsAny + readOnlyRootFilesystem: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - configMap + - emptyDir + - secret + - hostPath +{{- end }} +{{- end }} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke2/logging-rke2-containers.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke2/logging-rke2-containers.yaml new file mode 100755 index 000000000..7be4972e7 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/rke2/logging-rke2-containers.yaml @@ -0,0 +1,73 @@ +{{- if .Values.additionalLoggingSources.rke2.enabled }} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Logging +metadata: + name: {{ .Release.Name }}-rke2-containers + namespace: {{ .Release.Namespace }} +spec: + controlNamespace: {{ .Release.Namespace }} + fluentbit: + image: + repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentbit.repository }} + tag: {{ .Values.images.fluentbit.tag }} + inputTail: + Tag: "rke2" + Path: "/var/log/containers/*rke*.log" + extraVolumeMounts: + - source: "/var/log/containers/" + destination: "/var/log/containers/" + readOnly: true + {{- if or .Values.global.psp.enabled .Values.global.seLinux.enabled }} + security: + {{- end }} + {{- if or .Values.global.psp.enabled }} + podSecurityPolicyCreate: true + roleBasedAccessControlCreate: true + {{- end }} + {{- if .Values.global.seLinux.enabled }} + securityContext: + seLinuxOptions: + type: rke_logreader_t + {{- end }} + {{- $total_tolerations := concat (.Values.tolerations) (.Values.fluentbit.tolerations) }} + {{- with $total_tolerations }} + tolerations: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentbit.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + fluentd: + image: + repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentd.repository }} + tag: {{ .Values.images.fluentd.tag }} + configReloaderImage: + repository: {{ template "system_default_registry" . }}{{ .Values.images.config_reloader.repository }} + tag: {{ .Values.images.config_reloader.tag }} + disablePvc: {{ .Values.disablePvc }} + {{- if .Values.global.psp.enabled }} + security: + podSecurityPolicyCreate: true + roleBasedAccessControlCreate: true + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentd.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentd.livenessProbe }} + livenessProbe: {{- toYaml . | nindent 6 }} + {{- end }} +{{- end }} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/root/logging.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/root/logging.yaml new file mode 100755 index 000000000..b7581cad3 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/loggings/root/logging.yaml @@ -0,0 +1,74 @@ +{{- $containers := printf "%s/containers/" (default "/var/lib/docker" .Values.global.dockerRootDirectory) }} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Logging +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +spec: + controlNamespace: {{ .Release.Namespace }} + fluentbit: + image: + repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentbit.repository }} + tag: {{ .Values.images.fluentbit.tag }} + {{- if or .Values.global.psp.enabled .Values.global.seLinux.enabled }} + security: + {{- end }} + {{- if .Values.global.psp.enabled }} + podSecurityPolicyCreate: true + roleBasedAccessControlCreate: true + {{- end }} + {{- if .Values.global.seLinux.enabled }} + securityContext: + seLinuxOptions: + type: rke_logreader_t + {{- end }} + {{- if .Values.global.dockerRootDirectory }} + mountPath: {{ $containers }} + extraVolumeMounts: + - source: {{ $containers }} + destination: {{ $containers }} + readOnly: true + {{- end }} + {{- $total_tolerations := concat (.Values.tolerations) (.Values.fluentbit.tolerations) }} + {{- with $total_tolerations }} + tolerations: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentbit.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + fluentd: + image: + repository: {{ template "system_default_registry" . }}{{ .Values.images.fluentd.repository }} + tag: {{ .Values.images.fluentd.tag }} + configReloaderImage: + repository: {{ template "system_default_registry" . }}{{ .Values.images.config_reloader.repository }} + tag: {{ .Values.images.config_reloader.tag }} + disablePvc: {{ .Values.disablePvc }} + {{- if .Values.global.psp.enabled }} + security: + podSecurityPolicyCreate: true + roleBasedAccessControlCreate: true + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentd.resources }} + resources: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.fluentd.livenessProbe }} + livenessProbe: {{- toYaml . | nindent 6 }} + {{- end }} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/psp.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/psp.yaml new file mode 100755 index 000000000..d0eab300c --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/psp.yaml @@ -0,0 +1,33 @@ +{{ if and .Values.rbac.enabled .Values.rbac.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: psp.logging-operator + namespace: {{ include "logging-operator.namespace" . }} + annotations: + seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default,runtime/default' + seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default,runtime/default' + labels: +{{ include "logging-operator.labels" . | indent 4 }} +spec: + readOnlyRootFilesystem: true + privileged: false + allowPrivilegeEscalation: false + runAsUser: + rule: MustRunAsNonRoot + fsGroup: + rule: MustRunAs + ranges: + - min: 1 + max: 65535 + supplementalGroups: + rule: MustRunAs + ranges: + - min: 1 + max: 65535 + seLinux: + rule: RunAsAny + volumes: + - secret + - configMap +{{ end }} \ No newline at end of file diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/service.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/service.yaml new file mode 100755 index 000000000..f419ae2c4 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "logging-operator.fullname" . }} + namespace: {{ include "logging-operator.namespace" . }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +spec: + type: ClusterIP + {{- with .Values.http.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + ports: + - port: {{ .Values.http.port }} + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: {{ include "logging-operator.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/serviceMonitor.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/serviceMonitor.yaml new file mode 100755 index 000000000..1bb762cde --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/serviceMonitor.yaml @@ -0,0 +1,30 @@ +{{ if .Values.monitoring.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "logging-operator.fullname" . }} + namespace: {{ include "logging-operator.namespace" . }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +{{- with .Values.monitoring.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} +{{- end }} +spec: + selector: + matchLabels: +{{ include "logging-operator.labels" . | indent 6 }} + endpoints: + - port: http + path: /metrics + {{- with .Values.monitoring.serviceMonitor.metricsRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- with .Values.monitoring.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + namespaceSelector: + matchNames: + - {{ include "logging-operator.namespace" . }} +{{- end }} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/serviceaccount.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/serviceaccount.yaml new file mode 100755 index 000000000..cbb2a94b4 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/serviceaccount.yaml @@ -0,0 +1,10 @@ +{{- if .Values.rbac.enabled }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "logging-operator.fullname" . }} + namespace: {{ include "logging-operator.namespace" . }} + labels: +{{ include "logging-operator.labels" . | indent 4 }} +{{- end }} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/userroles.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/userroles.yaml new file mode 100755 index 000000000..f4136b09a --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/userroles.yaml @@ -0,0 +1,35 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: "logging-admin" + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" +rules: + - apiGroups: + - "logging.banzaicloud.io" + resources: + - flows + - outputs + verbs: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: "logging-view" + labels: + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" +rules: + - apiGroups: + - "logging.banzaicloud.io" + resources: + - flows + - outputs + - clusterflows + - clusteroutputs + verbs: + - get + - list + - watch diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/validate-install-crd.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/validate-install-crd.yaml new file mode 100755 index 000000000..66e8725e5 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/validate-install-crd.yaml @@ -0,0 +1,18 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "logging.banzaicloud.io/v1beta1/ClusterFlow" false -}} +# {{- set $found "logging.banzaicloud.io/v1beta1/ClusterOutput" false -}} +# {{- set $found "logging.banzaicloud.io/v1beta1/Flow" false -}} +# {{- set $found "logging.banzaicloud.io/v1beta1/Logging" false -}} +# {{- set $found "logging.banzaicloud.io/v1beta1/Output" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install the corresponding CRD chart before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} \ No newline at end of file diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/validate-install.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/validate-install.yaml new file mode 100755 index 000000000..bd624cc4b --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/templates/validate-install.yaml @@ -0,0 +1,5 @@ +#{{- if .Values.global.dockerRootDirectory }} +#{{- if or (hasSuffix "/containers" .Values.global.dockerRootDirectory) (hasSuffix "/" .Values.global.dockerRootDirectory) }} +#{{- required "global.dockerRootDirectory must not end with suffix: '/' or '/containers'" "" -}} +#{{- end }} +#{{- end }} diff --git a/charts/rancher-logging/rancher-logging/3.9.002-rc03/values.yaml b/charts/rancher-logging/rancher-logging/3.9.002-rc03/values.yaml new file mode 100755 index 000000000..fab44ef84 --- /dev/null +++ b/charts/rancher-logging/rancher-logging/3.9.002-rc03/values.yaml @@ -0,0 +1,156 @@ +# Default values for logging-operator. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: rancher/mirrored-banzaicloud-logging-operator + tag: 3.9.0 + pullPolicy: IfNotPresent + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" +namespaceOverride: "" + +annotations: {} + +## Deploy CRDs used by Logging Operator. +## +createCustomResource: false + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: + kubernetes.io/os: linux + +tolerations: + - key: cattle.io/os + operator: "Equal" + value: "linux" + effect: NoSchedule + +affinity: {} + +http: + # http listen port number + port: 8080 + # Service definition for query http service + service: + type: ClusterIP + clusterIP: None + # Annotations to query http service + annotations: {} + # Labels to query http service + labels: {} + +# These "rbac" settings match the upstream defaults. For only using psp in the overlay files, which +# include the default Logging CRs created, see the "global.psp" setting. To enable psp for the entire +# chart, enable both "rbac.psp" and "global.psp" (this may require further changes to the chart). +rbac: + enabled: true + psp: + enabled: false + +## SecurityContext holds pod-level security attributes and common container settings. +## This defaults to non root user with uid 1000 and gid 2000. *v1.PodSecurityContext false +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ +## +podSecurityContext: {} +# runAsNonRoot: true +# runAsUser: 1000 +# fsGroup: 2000 +securityContext: {} +# allowPrivilegeEscalation: false +# readOnlyRootFilesystem: true + # capabilities: + # drop: ["ALL"] + +## Operator priorityClassName +## +priorityClassName: {} + +monitoring: + # Create a Prometheus Operator ServiceMonitor object + serviceMonitor: + enabled: false + additionalLabels: {} + metricRelabelings: [] + relabelings: [] + +disablePvc: true + +additionalLoggingSources: + rke: + enabled: false + fluentbit: + log_level: "info" + mem_buffer_limit: "5MB" + rke2: + enabled: false + k3s: + enabled: false + container_engine: "systemd" + aks: + enabled: false + eks: + enabled: false + gke: + enabled: false + +images: + config_reloader: + repository: rancher/mirrored-jimmidyson-configmap-reload + tag: v0.4.0 + fluentbit: + repository: rancher/mirrored-fluent-fluent-bit + tag: 1.6.10 + fluentbit_debug: + repository: rancher/mirrored-fluent-fluent-bit + tag: 1.6.10-debug + fluentd: + repository: rancher/mirrored-banzaicloud-fluentd + tag: v1.11.5-alpine-9 + +# These "fluentd" and "fluentbit" settings apply to every Logging CR, including vendor Logging CRs +# enabled in "additionalLoggingSources". Changing these affects every Logging CR installed. +fluentd: + resources: {} + livenessProbe: + tcpSocket: + port: 24240 + initialDelaySeconds: 30 + periodSeconds: 15 +fluentbit: + resources: {} + tolerations: + - key: node-role.kubernetes.io/controlplane + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/etcd + value: "true" + effect: NoExecute + +global: + cattle: + systemDefaultRegistry: "" + # Change the "dockerRootDirectory" if the default Docker directory has changed. + dockerRootDirectory: "" + # This psp setting differs from the upstream "rbac.psp" by only enabling psp settings for the + # overlay files, which include the Logging CRs created, whereas the upstream "rbac.psp" affects the + # logging operator. + psp: + enabled: true + seLinux: + enabled: false diff --git a/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/Chart.yaml new file mode 100755 index 000000000..87b7bb474 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/Chart.yaml @@ -0,0 +1,10 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/hidden: "true" + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/release-name: rancher-monitoring-crd +apiVersion: v1 +description: Installs the CRDs for rancher-monitoring. +name: rancher-monitoring-crd +type: application +version: 9.4.204-rc09 diff --git a/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/README.md b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/README.md new file mode 100755 index 000000000..48d2a8621 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/README.md @@ -0,0 +1,2 @@ +# rancher-monitoring-crd +A Rancher chart that installs the CRDs used by rancher-monitoring. diff --git a/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-alertmanager.yaml b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-alertmanager.yaml new file mode 100755 index 000000000..98030b4f8 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-alertmanager.yaml @@ -0,0 +1,4500 @@ +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.38/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.4 + creationTimestamp: null + name: alertmanagers.monitoring.coreos.com +spec: + additionalPrinterColumns: + - JSONPath: .spec.version + description: The version of Alertmanager + name: Version + type: string + - JSONPath: .spec.replicas + description: The desired replicas number of Alertmanagers + name: Replicas + type: integer + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: monitoring.coreos.com + names: + kind: Alertmanager + listKind: AlertmanagerList + plural: alertmanagers + singular: alertmanager + preserveUnknownFields: false + scope: Namespaced + subresources: {} + validation: + openAPIV3Schema: + description: Alertmanager describes an Alertmanager cluster. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: 'Specification of the desired behavior of the Alertmanager + cluster. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' + properties: + additionalPeers: + description: AdditionalPeers allows injecting a set of additional Alertmanagers + to peer with to form a highly available cluster. + items: + type: string + type: array + affinity: + description: If specified, the pod's scheduling constraints. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes + that satisfy the affinity expressions specified by this field, + but it may choose a node that violates one or more of the + expressions. The node that is most preferred is the one with + the greatest sum of weights, i.e. for each node that meets + all of the scheduling requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating through + the elements of this field and adding "weight" to the sum + if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: An empty preferred scheduling term matches all + objects with implicit weight 0 (i.e. it's a no-op). A null + preferred scheduling term matches no objects (i.e. is also + a no-op). + properties: + preference: + description: A node selector term, associated with the + corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the + operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be + empty. If the operator is Gt or Lt, the values + array must have a single element, which will + be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the + operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be + empty. If the operator is Gt or Lt, the values + array must have a single element, which will + be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by this + field are not met at scheduling time, the pod will not be + scheduled onto the node. If the affinity requirements specified + by this field cease to be met at some point during pod execution + (e.g. due to an update), the system may or may not try to + eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The + terms are ORed. + items: + description: A null or empty node selector term matches + no objects. The requirements of them are ANDed. The + TopologySelectorTerm type implements a subset of the + NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the + operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be + empty. If the operator is Gt or Lt, the values + array must have a single element, which will + be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the + operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be + empty. If the operator is Gt or Lt, the values + array must have a single element, which will + be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes + that satisfy the affinity expressions specified by this field, + but it may choose a node that violates one or more of the + expressions. The node that is most preferred is the one with + the greatest sum of weights, i.e. for each node that meets + all of the scheduling requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating through + the elements of this field and adding "weight" to the sum + if the node has pods which matches the corresponding podAffinityTerm; + the node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement is + a selector that contains values, a key, and + an operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. This array + is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified namespaces, + where co-located is defined as running on a node + whose value of the label with key topologyKey matches + that of any node on which any of the selected pods + is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the corresponding + podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by this + field are not met at scheduling time, the pod will not be + scheduled onto the node. If the affinity requirements specified + by this field cease to be met at some point during pod execution + (e.g. due to a pod label update), the system may or may not + try to eventually evict the pod from its node. When there + are multiple elements, the lists of nodes corresponding to + each podAffinityTerm are intersected, i.e. all terms must + be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) that + this pod should be co-located (affinity) or not co-located + (anti-affinity) with, where co-located is defined as running + on a node whose value of the label with key + matches that of any node on which a pod of the set of pods + is running + properties: + labelSelector: + description: A label query over a set of resources, in + this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values + array must be non-empty. If the operator is + Exists or DoesNotExist, the values array must + be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces the + labelSelector applies to (matches against); null or + empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where + co-located is defined as running on a node whose value + of the label with key topologyKey matches that of any + node on which any of the selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some other + pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes + that satisfy the anti-affinity expressions specified by this + field, but it may choose a node that violates one or more + of the expressions. The node that is most preferred is the + one with the greatest sum of weights, i.e. for each node that + meets all of the scheduling requirements (resource request, + requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field + and adding "weight" to the sum if the node has pods which + matches the corresponding podAffinityTerm; the node(s) with + the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement is + a selector that contains values, a key, and + an operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. This array + is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified namespaces, + where co-located is defined as running on a node + whose value of the label with key topologyKey matches + that of any node on which any of the selected pods + is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the corresponding + podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements specified by + this field are not met at scheduling time, the pod will not + be scheduled onto the node. If the anti-affinity requirements + specified by this field cease to be met at some point during + pod execution (e.g. due to a pod label update), the system + may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding + to each podAffinityTerm are intersected, i.e. all terms must + be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) that + this pod should be co-located (affinity) or not co-located + (anti-affinity) with, where co-located is defined as running + on a node whose value of the label with key + matches that of any node on which a pod of the set of pods + is running + properties: + labelSelector: + description: A label query over a set of resources, in + this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values + array must be non-empty. If the operator is + Exists or DoesNotExist, the values array must + be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces the + labelSelector applies to (matches against); null or + empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where + co-located is defined as running on a node whose value + of the label with key topologyKey matches that of any + node on which any of the selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + baseImage: + description: Base image that is used to deploy pods, without tag. + type: string + configMaps: + description: ConfigMaps is a list of ConfigMaps in the same namespace + as the Alertmanager object, which shall be mounted into the Alertmanager + Pods. The ConfigMaps are mounted into /etc/alertmanager/configmaps/. + items: + type: string + type: array + configSecret: + description: ConfigSecret is the name of a Kubernetes Secret in the + same namespace as the Alertmanager object, which contains configuration + for this Alertmanager instance. Defaults to 'alertmanager-' + The secret is mounted into /etc/alertmanager/config. + type: string + containers: + description: Containers allows injecting additional containers. This + is meant to allow adding an authentication proxy to an Alertmanager + pod. + items: + description: A single application container that you want to run within + a pod. + properties: + args: + description: 'Arguments to the entrypoint. The docker image''s + CMD is used if this is not provided. Variable references $(VAR_NAME) + are expanded using the container''s environment. If a variable + cannot be resolved, the reference in the input string will be + unchanged. The $(VAR_NAME) syntax can be escaped with a double + $$, ie: $$(VAR_NAME). Escaped references will never be expanded, + regardless of whether the variable exists or not. Cannot be + updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within a shell. The + docker image''s ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, the reference + in the input string will be unchanged. The $(VAR_NAME) syntax + can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable exists + or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previous defined environment variables in the + container and any service environment variables. If a + variable cannot be resolved, the reference in the input + string will be unchanged. The $(VAR_NAME) syntax can be + escaped with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable + exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, metadata.labels, metadata.annotations, + spec.nodeName, spec.serviceAccountName, status.hostIP, + status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + description: Specifies the output format of the + exposed resources, defaults to "1" + type: string + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment variables + in the container. The keys defined within a source must be a + C_IDENTIFIER. All invalid keys will be reported as an event + when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take + precedence. Values defined by an Env with a duplicate key will + take precedence. Cannot be updated. + items: + description: EnvFromSource represents the source of a set of + ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + prefix: + description: An optional identifier to prepend to each key + in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + type: object + type: array + image: + description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management + to default or override container images in workload controllers + like Deployments and StatefulSets.' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent + otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Actions that the management system should take in + response to container lifecycle events. Cannot be updated. + properties: + postStart: + description: 'PostStart is called immediately after a container + is created. If the handler fails, the container is terminated + and restarted according to its restart policy. Other management + of the container blocks until the hook completes. More info: + https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's filesystem. + The command is simply exec'd, it is not run inside + a shell, so traditional shell instructions ('|', + etc) won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is treated + as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately before a container + is terminated due to an API request or management event + such as liveness/startup probe failure, preemption, resource + contention, etc. The handler is not called if the container + crashes or exits. The reason for termination is passed to + the handler. The Pod''s termination grace period countdown + begins before the PreStop hooked is executed. Regardless + of the outcome of the handler, the container will eventually + terminate within the Pod''s termination grace period. Other + management of the container blocks until the hook completes + or until the termination grace period is reached. More info: + https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's filesystem. + The command is simply exec'd, it is not run inside + a shell, so traditional shell instructions ('|', + etc) won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is treated + as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: 'Periodic probe of container liveness. Container + will be restarted if the probe fails. Cannot be updated. More + info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + name: + description: Name of the container specified as a DNS_LABEL. Each + container in a pod must have a unique name (DNS_LABEL). Cannot + be updated. + type: string + ports: + description: List of ports to expose from the container. Exposing + a port here gives the system additional information about the + network connections a container uses, but is primarily informational. + Not specifying a port here DOES NOT prevent that port from being + exposed. Any port which is listening on the default "0.0.0.0" + address inside a container will be accessible from the network. + Cannot be updated. + items: + description: ContainerPort represents a network port in a single + container. + properties: + containerPort: + description: Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: Number of port to expose on the host. If specified, + this must be a valid port number, 0 < x < 65536. If HostNetwork + is specified, this must match ContainerPort. Most containers + do not need this. + format: int32 + type: integer + name: + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port in a pod must + have a unique name. Name for the port that can be referred + to by services. + type: string + protocol: + description: Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + readinessProbe: + description: 'Periodic probe of container service readiness. Container + will be removed from service endpoints if the probe fails. Cannot + be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + resources: + description: 'Compute Resources required by this container. Cannot + be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + properties: + limits: + additionalProperties: + type: string + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + type: string + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + securityContext: + description: 'Security options the pod should run with. More info: + https://kubernetes.io/docs/concepts/policy/security-context/ + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether a + process can gain more privileges than its parent process. + This bool directly controls if the no_new_privs flag will + be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as Privileged + 2) has CAP_SYS_ADMIN' + type: boolean + capabilities: + description: The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the + container runtime. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes in + privileged containers are essentially equivalent to root + on the host. Defaults to false. + type: boolean + procMount: + description: procMount denotes the type of proc mount to use + for the containers. The default is DefaultProcMount which + uses the container runtime defaults for readonly paths and + masked paths. This requires the ProcMountType feature flag + to be enabled. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only root filesystem. + Default is false. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container + process. Uses runtime default if unset. May also be set + in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a non-root + user. If true, the Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 (root) and fail + to start the container if it does. If unset or false, no + such validation will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container + process. Defaults to user specified in image metadata if + unspecified. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random + SELinux context for each container. May also be set in + PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + windowsOptions: + description: The Windows specific settings applied to all + containers. If unspecified, the options from the PodSecurityContext + will be used. If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission + webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec named + by the GMSACredentialSpecName field. This field is alpha-level + and is only honored by servers that enable the WindowsGMSA + feature flag. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. This field is alpha-level + and is only honored by servers that enable the WindowsGMSA + feature flag. + type: string + runAsUserName: + description: The UserName in Windows to run the entrypoint + of the container process. Defaults to the user specified + in image metadata if unspecified. May also be set in + PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. This field is beta-level and may be + disabled with the WindowsRunAsUserName feature flag. + type: string + type: object + type: object + startupProbe: + description: 'StartupProbe indicates that the Pod has successfully + initialized. If specified, no other probes are executed until + this completes successfully. If this probe fails, the Pod will + be restarted, just as if the livenessProbe failed. This can + be used to provide different probe parameters at the beginning + of a Pod''s lifecycle, when it might take a long time to load + data or warm a cache, than during steady-state operation. This + cannot be updated. This is an alpha feature enabled by the StartupProbe + feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + stdin: + description: Whether this container should allocate a buffer for + stdin in the container runtime. If this is not set, reads from + stdin in the container will always result in EOF. Default is + false. + type: boolean + stdinOnce: + description: Whether the container runtime should close the stdin + channel after it has been opened by a single attach. When stdin + is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container + start, is empty until the first client attaches to stdin, and + then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container + is restarted. If this flag is false, a container processes that + reads from stdin will never receive an EOF. Default is false + type: boolean + terminationMessagePath: + description: 'Optional: Path at which the file to which the container''s + termination message will be written is mounted into the container''s + filesystem. Message written is intended to be brief final status, + such as an assertion failure message. Will be truncated by the + node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. Defaults to /dev/termination-log. + Cannot be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message should be populated. + File will use the contents of terminationMessagePath to populate + the container status message on both success and failure. FallbackToLogsOnError + will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever + is smaller. Defaults to File. Cannot be updated. + type: string + tty: + description: Whether this container should allocate a TTY for + itself, also requires 'stdin' to be true. Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be + used by the container. This is a beta feature. + items: + description: volumeDevice describes a mapping of a raw block + device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container + that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: Path within the container at which the volume + should be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts are + propagated from the host to container and the other way + around. When not set, MountPropagationNone is used. This + field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the container's + volume should be mounted. Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from which + the container's volume should be mounted. Behaves similarly + to SubPath but environment variable references $(VAR_NAME) + are expanded using the container's environment. Defaults + to "" (volume's root). SubPathExpr and SubPath are mutually + exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If not specified, + the container runtime's default will be used, which might be + configured in the container image. Cannot be updated. + type: string + required: + - name + type: object + type: array + externalUrl: + description: The external URL the Alertmanager instances will be available + under. This is necessary to generate correct URLs. This is necessary + if Alertmanager is not served from root of a DNS name. + type: string + image: + description: Image if specified has precedence over baseImage, tag and + sha combinations. Specifying the version is still necessary to ensure + the Prometheus Operator knows what version of Alertmanager is being + configured. + type: string + imagePullSecrets: + description: An optional list of references to secrets in the same namespace + to use for pulling prometheus and alertmanager images from registries + see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod + items: + description: LocalObjectReference contains enough information to let + you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + type: array + initContainers: + description: 'InitContainers allows adding initContainers to the pod + definition. Those can be used to e.g. fetch secrets for injection + into the Alertmanager configuration from external sources. Any errors + during the execution of an initContainer will lead to a restart of + the Pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + Using initContainers for any use case other then secret fetching is + entirely outside the scope of what the maintainers will support and + by doing so, you accept that this behaviour may break at any time + without notice.' + items: + description: A single application container that you want to run within + a pod. + properties: + args: + description: 'Arguments to the entrypoint. The docker image''s + CMD is used if this is not provided. Variable references $(VAR_NAME) + are expanded using the container''s environment. If a variable + cannot be resolved, the reference in the input string will be + unchanged. The $(VAR_NAME) syntax can be escaped with a double + $$, ie: $$(VAR_NAME). Escaped references will never be expanded, + regardless of whether the variable exists or not. Cannot be + updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within a shell. The + docker image''s ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, the reference + in the input string will be unchanged. The $(VAR_NAME) syntax + can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable exists + or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previous defined environment variables in the + container and any service environment variables. If a + variable cannot be resolved, the reference in the input + string will be unchanged. The $(VAR_NAME) syntax can be + escaped with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable + exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, metadata.labels, metadata.annotations, + spec.nodeName, spec.serviceAccountName, status.hostIP, + status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + description: Specifies the output format of the + exposed resources, defaults to "1" + type: string + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment variables + in the container. The keys defined within a source must be a + C_IDENTIFIER. All invalid keys will be reported as an event + when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take + precedence. Values defined by an Env with a duplicate key will + take precedence. Cannot be updated. + items: + description: EnvFromSource represents the source of a set of + ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + prefix: + description: An optional identifier to prepend to each key + in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + type: object + type: array + image: + description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management + to default or override container images in workload controllers + like Deployments and StatefulSets.' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent + otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Actions that the management system should take in + response to container lifecycle events. Cannot be updated. + properties: + postStart: + description: 'PostStart is called immediately after a container + is created. If the handler fails, the container is terminated + and restarted according to its restart policy. Other management + of the container blocks until the hook completes. More info: + https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's filesystem. + The command is simply exec'd, it is not run inside + a shell, so traditional shell instructions ('|', + etc) won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is treated + as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately before a container + is terminated due to an API request or management event + such as liveness/startup probe failure, preemption, resource + contention, etc. The handler is not called if the container + crashes or exits. The reason for termination is passed to + the handler. The Pod''s termination grace period countdown + begins before the PreStop hooked is executed. Regardless + of the outcome of the handler, the container will eventually + terminate within the Pod''s termination grace period. Other + management of the container blocks until the hook completes + or until the termination grace period is reached. More info: + https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's filesystem. + The command is simply exec'd, it is not run inside + a shell, so traditional shell instructions ('|', + etc) won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is treated + as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: 'Periodic probe of container liveness. Container + will be restarted if the probe fails. Cannot be updated. More + info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + name: + description: Name of the container specified as a DNS_LABEL. Each + container in a pod must have a unique name (DNS_LABEL). Cannot + be updated. + type: string + ports: + description: List of ports to expose from the container. Exposing + a port here gives the system additional information about the + network connections a container uses, but is primarily informational. + Not specifying a port here DOES NOT prevent that port from being + exposed. Any port which is listening on the default "0.0.0.0" + address inside a container will be accessible from the network. + Cannot be updated. + items: + description: ContainerPort represents a network port in a single + container. + properties: + containerPort: + description: Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: Number of port to expose on the host. If specified, + this must be a valid port number, 0 < x < 65536. If HostNetwork + is specified, this must match ContainerPort. Most containers + do not need this. + format: int32 + type: integer + name: + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port in a pod must + have a unique name. Name for the port that can be referred + to by services. + type: string + protocol: + description: Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + readinessProbe: + description: 'Periodic probe of container service readiness. Container + will be removed from service endpoints if the probe fails. Cannot + be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + resources: + description: 'Compute Resources required by this container. Cannot + be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + properties: + limits: + additionalProperties: + type: string + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + type: string + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + securityContext: + description: 'Security options the pod should run with. More info: + https://kubernetes.io/docs/concepts/policy/security-context/ + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether a + process can gain more privileges than its parent process. + This bool directly controls if the no_new_privs flag will + be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as Privileged + 2) has CAP_SYS_ADMIN' + type: boolean + capabilities: + description: The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the + container runtime. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes in + privileged containers are essentially equivalent to root + on the host. Defaults to false. + type: boolean + procMount: + description: procMount denotes the type of proc mount to use + for the containers. The default is DefaultProcMount which + uses the container runtime defaults for readonly paths and + masked paths. This requires the ProcMountType feature flag + to be enabled. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only root filesystem. + Default is false. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container + process. Uses runtime default if unset. May also be set + in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a non-root + user. If true, the Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 (root) and fail + to start the container if it does. If unset or false, no + such validation will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container + process. Defaults to user specified in image metadata if + unspecified. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random + SELinux context for each container. May also be set in + PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + windowsOptions: + description: The Windows specific settings applied to all + containers. If unspecified, the options from the PodSecurityContext + will be used. If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission + webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec named + by the GMSACredentialSpecName field. This field is alpha-level + and is only honored by servers that enable the WindowsGMSA + feature flag. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. This field is alpha-level + and is only honored by servers that enable the WindowsGMSA + feature flag. + type: string + runAsUserName: + description: The UserName in Windows to run the entrypoint + of the container process. Defaults to the user specified + in image metadata if unspecified. May also be set in + PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. This field is beta-level and may be + disabled with the WindowsRunAsUserName feature flag. + type: string + type: object + type: object + startupProbe: + description: 'StartupProbe indicates that the Pod has successfully + initialized. If specified, no other probes are executed until + this completes successfully. If this probe fails, the Pod will + be restarted, just as if the livenessProbe failed. This can + be used to provide different probe parameters at the beginning + of a Pod''s lifecycle, when it might take a long time to load + data or warm a cache, than during steady-state operation. This + cannot be updated. This is an alpha feature enabled by the StartupProbe + feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + stdin: + description: Whether this container should allocate a buffer for + stdin in the container runtime. If this is not set, reads from + stdin in the container will always result in EOF. Default is + false. + type: boolean + stdinOnce: + description: Whether the container runtime should close the stdin + channel after it has been opened by a single attach. When stdin + is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container + start, is empty until the first client attaches to stdin, and + then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container + is restarted. If this flag is false, a container processes that + reads from stdin will never receive an EOF. Default is false + type: boolean + terminationMessagePath: + description: 'Optional: Path at which the file to which the container''s + termination message will be written is mounted into the container''s + filesystem. Message written is intended to be brief final status, + such as an assertion failure message. Will be truncated by the + node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. Defaults to /dev/termination-log. + Cannot be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message should be populated. + File will use the contents of terminationMessagePath to populate + the container status message on both success and failure. FallbackToLogsOnError + will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever + is smaller. Defaults to File. Cannot be updated. + type: string + tty: + description: Whether this container should allocate a TTY for + itself, also requires 'stdin' to be true. Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be + used by the container. This is a beta feature. + items: + description: volumeDevice describes a mapping of a raw block + device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container + that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: Path within the container at which the volume + should be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts are + propagated from the host to container and the other way + around. When not set, MountPropagationNone is used. This + field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the container's + volume should be mounted. Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from which + the container's volume should be mounted. Behaves similarly + to SubPath but environment variable references $(VAR_NAME) + are expanded using the container's environment. Defaults + to "" (volume's root). SubPathExpr and SubPath are mutually + exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If not specified, + the container runtime's default will be used, which might be + configured in the container image. Cannot be updated. + type: string + required: + - name + type: object + type: array + listenLocal: + description: ListenLocal makes the Alertmanager server listen on loopback, + so that it does not bind against the Pod IP. Note this is only for + the Alertmanager UI, not the gossip communication. + type: boolean + logFormat: + description: Log format for Alertmanager to be configured with. + type: string + logLevel: + description: Log level for Alertmanager to be configured with. + type: string + nodeSelector: + additionalProperties: + type: string + description: Define which Nodes the Pods are scheduled on. + type: object + paused: + description: If set to true all actions on the underlaying managed objects + are not goint to be performed, except for delete actions. + type: boolean + podMetadata: + description: PodMetadata configures Labels and Annotations which are + propagated to the alertmanager pods. + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value map stored + with a resource that may be set by external tools to store and + retrieve arbitrary metadata. They are not queryable and should + be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations' + type: object + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can be used to + organize and categorize (scope and select) objects. May match + selectors of replication controllers and services. More info: + http://kubernetes.io/docs/user-guide/labels' + type: object + type: object + portName: + description: Port name used for the pods and governing service. This + defaults to web + type: string + priorityClassName: + description: Priority class assigned to the Pods + type: string + replicas: + description: Size is the expected size of the alertmanager cluster. + The controller will eventually make the size of the running cluster + equal to the expected size. + format: int32 + type: integer + resources: + description: Define resources requests and limits for single Pods. + properties: + limits: + additionalProperties: + type: string + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + type: string + description: 'Requests describes the minimum amount of compute resources + required. If Requests is omitted for a container, it defaults + to Limits if that is explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + retention: + description: Time duration Alertmanager shall retain data for. Default + is '120h', and must match the regular expression `[0-9]+(ms|s|m|h)` + (milliseconds seconds minutes hours). + type: string + routePrefix: + description: The route prefix Alertmanager registers HTTP handlers for. + This is useful, if using ExternalURL and a proxy is rewriting HTTP + routes of a request, and the actual ExternalURL is still true, but + the server serves requests under a different route prefix. For example + for use with `kubectl proxy`. + type: string + secrets: + description: Secrets is a list of Secrets in the same namespace as the + Alertmanager object, which shall be mounted into the Alertmanager + Pods. The Secrets are mounted into /etc/alertmanager/secrets/. + items: + type: string + type: array + securityContext: + description: SecurityContext holds pod-level security attributes and + common container settings. This defaults to the default PodSecurityContext. + properties: + fsGroup: + description: "A special supplemental group that applies to all containers + in a pod. Some volume types allow the Kubelet to change the ownership + of that volume to be owned by the pod: \n 1. The owning GID will + be the FSGroup 2. The setgid bit is set (new files created in + the volume will be owned by FSGroup) 3. The permission bits are + OR'd with rw-rw---- \n If unset, the Kubelet will not modify the + ownership and permissions of any volume." + format: int64 + type: integer + runAsGroup: + description: The GID to run the entrypoint of the container process. + Uses runtime default if unset. May also be set in SecurityContext. If + set in both SecurityContext and PodSecurityContext, the value + specified in SecurityContext takes precedence for that container. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a non-root + user. If true, the Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 (root) and fail to start + the container if it does. If unset or false, no such validation + will be performed. May also be set in SecurityContext. If set + in both SecurityContext and PodSecurityContext, the value specified + in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. May + also be set in SecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux + context for each container. May also be set in SecurityContext. If + set in both SecurityContext and PodSecurityContext, the value + specified in SecurityContext takes precedence for that container. + properties: + level: + description: Level is SELinux level label that applies to the + container. + type: string + role: + description: Role is a SELinux role label that applies to the + container. + type: string + type: + description: Type is a SELinux type label that applies to the + container. + type: string + user: + description: User is a SELinux user label that applies to the + container. + type: string + type: object + supplementalGroups: + description: A list of groups applied to the first process run in + each container, in addition to the container's primary GID. If + unspecified, no groups will be added to any container. + items: + format: int64 + type: integer + type: array + sysctls: + description: Sysctls hold a list of namespaced sysctls used for + the pod. Pods with unsupported sysctls (by the container runtime) + might fail to launch. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + description: The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext + will be used. If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission + webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec named by + the GMSACredentialSpecName field. This field is alpha-level + and is only honored by servers that enable the WindowsGMSA + feature flag. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA + credential spec to use. This field is alpha-level and is only + honored by servers that enable the WindowsGMSA feature flag. + type: string + runAsUserName: + description: The UserName in Windows to run the entrypoint of + the container process. Defaults to the user specified in image + metadata if unspecified. May also be set in PodSecurityContext. + If set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. This + field is beta-level and may be disabled with the WindowsRunAsUserName + feature flag. + type: string + type: object + type: object + serviceAccountName: + description: ServiceAccountName is the name of the ServiceAccount to + use to run the Prometheus Pods. + type: string + sha: + description: SHA of Alertmanager container image to be deployed. Defaults + to the value of `version`. Similar to a tag, but the SHA explicitly + deploys an immutable container image. Version and Tag are ignored + if SHA is set. + type: string + storage: + description: Storage is the definition of how storage will be used by + the Alertmanager instances. + properties: + emptyDir: + description: 'EmptyDirVolumeSource to be used by the Prometheus + StatefulSets. If specified, used in place of any volumeClaimTemplate. + More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir' + properties: + medium: + description: 'What type of storage medium should back this directory. + The default is "" which means to use the node''s default medium. + Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + description: 'Total amount of local storage required for this + EmptyDir volume. The size limit is also applicable for memory + medium. The maximum usage on memory medium EmptyDir would + be the minimum value between the SizeLimit specified here + and the sum of memory limits of all containers in a pod. The + default is nil which means that the limit is undefined. More + info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + type: string + type: object + volumeClaimTemplate: + description: A PVC spec to be used by the Prometheus StatefulSets. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this + representation of an object. Servers should convert recognized + schemas to the latest internal value, and may reject unrecognized + values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint + the client submits requests to. Cannot be updated. In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' + type: object + spec: + description: 'Spec defines the desired characteristics of a + volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'AccessModes contains the desired access modes + the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: This field requires the VolumeSnapshotDataSource + alpha feature gate to be enabled and currently VolumeSnapshot + is the only supported data source. If the provisioner + can support VolumeSnapshot data source, it will create + a new volume and data will be restored to the volume at + the same time. If the provisioner does not support VolumeSnapshot + data source, volume will not be created and the failure + will be reported as an event. In the future, we plan to + support more data source types and the behavior of the + provisioner may change. + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, the + specified Kind must be in the core API group. For + any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'Resources represents the minimum resources + the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + type: string + description: 'Limits describes the maximum amount of + compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + type: string + description: 'Requests describes the minimum amount + of compute resources required. If Requests is omitted + for a container, it defaults to Limits if that is + explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + selector: + description: A label query over volumes to consider for + binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + storageClassName: + description: 'Name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume is required + by the claim. Value of Filesystem is implied when not + included in claim spec. This is a beta feature. + type: string + volumeName: + description: VolumeName is the binding reference to the + PersistentVolume backing this claim. + type: string + type: object + status: + description: 'Status represents the current information/status + of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'AccessModes contains the actual access modes + the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + capacity: + additionalProperties: + type: string + description: Represents the actual resources of the underlying + volume. + type: object + conditions: + description: Current Condition of persistent volume claim. + If underlying persistent volume is being resized then + the Condition will be set to 'ResizeStarted'. + items: + description: PersistentVolumeClaimCondition contails details + about state of pvc + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transitioned + from one status to another. + format: date-time + type: string + message: + description: Human-readable message indicating details + about last transition. + type: string + reason: + description: Unique, this should be a short, machine + understandable string that gives the reason for + condition's last transition. If it reports "ResizeStarted" + that means the underlying persistent volume is being + resized. + type: string + status: + type: string + type: + description: PersistentVolumeClaimConditionType is + a valid value of PersistentVolumeClaimCondition.Type + type: string + required: + - status + - type + type: object + type: array + phase: + description: Phase represents the current phase of PersistentVolumeClaim. + type: string + type: object + type: object + type: object + tag: + description: Tag of Alertmanager container image to be deployed. Defaults + to the value of `version`. Version is ignored if Tag is set. + type: string + tolerations: + description: If specified, the pod's tolerations. + items: + description: The pod this Toleration is attached to tolerates any + taint that matches the triple using the matching + operator . + properties: + effect: + description: Effect indicates the taint effect to match. Empty + means match all taint effects. When specified, allowed values + are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies + to. Empty means match all taint keys. If the key is empty, operator + must be Exists; this combination means to match all values and + all keys. + type: string + operator: + description: Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. Exists + is equivalent to wildcard for value, so that a pod can tolerate + all taints of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of time the + toleration (which must be of effect NoExecute, otherwise this + field is ignored) tolerates the taint. By default, it is not + set, which means tolerate the taint forever (do not evict). + Zero and negative values will be treated as 0 (evict immediately) + by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise + just a regular string. + type: string + type: object + type: array + version: + description: Version the cluster should be on. + type: string + volumeMounts: + description: VolumeMounts allows configuration of additional VolumeMounts + on the output StatefulSet definition. VolumeMounts specified will + be appended to other VolumeMounts in the alertmanager container, that + are generated as a result of StorageSpec objects. + items: + description: VolumeMount describes a mounting of a Volume within a + container. + properties: + mountPath: + description: Path within the container at which the volume should + be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts are propagated + from the host to container and the other way around. When not + set, MountPropagationNone is used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise (false + or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the container's + volume should be mounted. Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from which the container's + volume should be mounted. Behaves similarly to SubPath but environment + variable references $(VAR_NAME) are expanded using the container's + environment. Defaults to "" (volume's root). SubPathExpr and + SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + volumes: + description: Volumes allows configuration of additional volumes on the + output StatefulSet definition. Volumes specified will be appended + to other volumes that are generated as a result of StorageSpec objects. + items: + description: Volume represents a named volume in a pod that may be + accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: 'AWSElasticBlockStore represents an AWS Disk resource + that is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + properties: + fsType: + description: 'Filesystem type of the volume that you want + to mount. Tip: Ensure that the filesystem type is supported + by the host operating system. Examples: "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem from compromising + the machine' + type: string + partition: + description: 'The partition in the volume that you want to + mount. If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition + as "1". Similarly, the volume partition for /dev/sda is + "0" (or you can leave the property empty).' + format: int32 + type: integer + readOnly: + description: 'Specify "true" to force and set the ReadOnly + property in VolumeMounts to "true". If omitted, the default + is "false". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'Unique ID of the persistent disk resource in + AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID + type: object + azureDisk: + description: AzureDisk represents an Azure Data Disk mount on + the host and bind mount to the pod. + properties: + cachingMode: + description: 'Host Caching mode: None, Read Only, Read Write.' + type: string + diskName: + description: The Name of the data disk in the blob storage + type: string + diskURI: + description: The URI the data disk in the blob storage + type: string + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'Expected values Shared: multiple blob disks + per storage account Dedicated: single blob disk per storage + account Managed: azure managed data disk (only in managed + availability set). defaults to shared' + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly here + will force the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: AzureFile represents an Azure File Service mount + on the host and bind mount to the pod. + properties: + readOnly: + description: Defaults to false (read/write). ReadOnly here + will force the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: the name of secret that contains Azure Storage + Account Name and Key + type: string + shareName: + description: Share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: CephFS represents a Ceph FS mount on the host that + shares a pod's lifetime + properties: + monitors: + description: 'Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'Optional: Used as the mounted root, rather than + the full Ceph tree, default is /' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. More + info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'Optional: SecretFile is the path to key ring + for User, default is /etc/ceph/user.secret More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'Optional: SecretRef is reference to the authentication + secret for User, default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + user: + description: 'Optional: User is the rados user name, default + is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors + type: object + cinder: + description: 'Cinder represents a cinder volume attached and mounted + on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Examples: "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. More + info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'Optional: points to a secret object containing + parameters used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + volumeID: + description: 'volume id used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + required: + - volumeID + type: object + configMap: + description: ConfigMap represents a configMap that should populate + this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created files + by default. Must be a value between 0 and 0777. Defaults + to 0644. Directories within the path are not affected by + this setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair in the Data + field of the referenced ConfigMap will be projected into + the volume as a file whose name is the key and content is + the value. If specified, the listed keys will be projected + into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the + ConfigMap, the volume setup will error unless it is marked + optional. Paths must be relative and may not contain the + '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to use on this file, + must be a value between 0 and 0777. If not specified, + the volume defaultMode will be used. This might be + in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode + bits set.' + format: int32 + type: integer + path: + description: The relative path of the file to map the + key to. May not be an absolute path. May not contain + the path element '..'. May not start with the string + '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its keys must + be defined + type: boolean + type: object + csi: + description: CSI (Container Storage Interface) represents storage + that is handled by an external CSI driver (Alpha feature). + properties: + driver: + description: Driver is the name of the CSI driver that handles + this volume. Consult with your admin for the correct name + as registered in the cluster. + type: string + fsType: + description: Filesystem type to mount. Ex. "ext4", "xfs", + "ntfs". If not provided, the empty value is passed to the + associated CSI driver which will determine the default filesystem + to apply. + type: string + nodePublishSecretRef: + description: NodePublishSecretRef is a reference to the secret + object containing sensitive information to pass to the CSI + driver to complete the CSI NodePublishVolume and NodeUnpublishVolume + calls. This field is optional, and may be empty if no secret + is required. If the secret object contains more than one + secret, all secret references are passed. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + readOnly: + description: Specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: VolumeAttributes stores driver-specific properties + that are passed to the CSI driver. Consult your driver's + documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: DownwardAPI represents downward API about the pod + that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created files + by default. Must be a value between 0 and 0777. Defaults + to 0644. Directories within the path are not affected by + this setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: + only annotations, labels, name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits to use on this file, + must be a value between 0 and 0777. If not specified, + the volume defaultMode will be used. This might be + in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode + bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative path name + of the file to be created. Must not be absolute or + contain the ''..'' path. Must be utf-8 encoded. The + first item of the relative path must not start with + ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + requests.cpu and requests.memory) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + description: Specifies the output format of the + exposed resources, defaults to "1" + type: string + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + description: 'EmptyDir represents a temporary directory that shares + a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'What type of storage medium should back this + directory. The default is "" which means to use the node''s + default medium. Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + description: 'Total amount of local storage required for this + EmptyDir volume. The size limit is also applicable for memory + medium. The maximum usage on memory medium EmptyDir would + be the minimum value between the SizeLimit specified here + and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + type: string + type: object + fc: + description: FC represents a Fibre Channel resource that is attached + to a kubelet's host machine and then exposed to the pod. + properties: + fsType: + description: 'Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + TODO: how do we prevent errors in the filesystem from compromising + the machine' + type: string + lun: + description: 'Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'Optional: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts.' + type: boolean + targetWWNs: + description: 'Optional: FC target worldwide names (WWNs)' + items: + type: string + type: array + wwids: + description: 'Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be + set, but not both simultaneously.' + items: + type: string + type: array + type: object + flexVolume: + description: FlexVolume represents a generic volume resource that + is provisioned/attached using an exec based plugin. + properties: + driver: + description: Driver is the name of the driver to use for this + volume. + type: string + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". The default filesystem depends on FlexVolume + script. + type: string + options: + additionalProperties: + type: string + description: 'Optional: Extra command options if any.' + type: object + readOnly: + description: 'Optional: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts.' + type: boolean + secretRef: + description: 'Optional: SecretRef is reference to the secret + object containing sensitive information to pass to the plugin + scripts. This may be empty if no secret object is specified. + If the secret object contains more than one secret, all + secrets are passed to the plugin scripts.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + required: + - driver + type: object + flocker: + description: Flocker represents a Flocker volume attached to a + kubelet's host machine. This depends on the Flocker control + service being running + properties: + datasetName: + description: Name of the dataset stored as metadata -> name + on the dataset for Flocker should be considered as deprecated + type: string + datasetUUID: + description: UUID of the dataset. This is unique identifier + of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'GCEPersistentDisk represents a GCE Disk resource + that is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'Filesystem type of the volume that you want + to mount. Tip: Ensure that the filesystem type is supported + by the host operating system. Examples: "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem from compromising + the machine' + type: string + partition: + description: 'The partition in the volume that you want to + mount. If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition + as "1". Similarly, the volume partition for /dev/sda is + "0" (or you can leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'Unique name of the PD resource in GCE. Used + to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'GitRepo represents a git repository at a particular + revision. DEPRECATED: GitRepo is deprecated. To provision a + container with a git repo, mount an EmptyDir into an InitContainer + that clones the repo using git, then mount the EmptyDir into + the Pod''s container.' + properties: + directory: + description: Target directory name. Must not contain or start + with '..'. If '.' is supplied, the volume directory will + be the git repository. Otherwise, if specified, the volume + will contain the git repository in the subdirectory with + the given name. + type: string + repository: + description: Repository URL + type: string + revision: + description: Commit hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'Glusterfs represents a Glusterfs mount on the host + that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'EndpointsName is the endpoint name that details + Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'Path is the Glusterfs volume path. More info: + https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'ReadOnly here will force the Glusterfs volume + to be mounted with read-only permissions. Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'HostPath represents a pre-existing file or directory + on the host machine that is directly exposed to the container. + This is generally used for system agents or other privileged + things that are allowed to see the host machine. Most containers + will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use host directory + mounts and who can/can not mount host directories as read/write.' + properties: + path: + description: 'Path of the directory on the host. If the path + is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'Type for HostPath Volume Defaults to "" More + info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'ISCSI represents an ISCSI Disk resource that is + attached to a kubelet''s host machine and then exposed to the + pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: whether support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: whether support iSCSI Session CHAP authentication + type: boolean + fsType: + description: 'Filesystem type of the volume that you want + to mount. Tip: Ensure that the filesystem type is supported + by the host operating system. Examples: "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem from compromising + the machine' + type: string + initiatorName: + description: Custom iSCSI Initiator Name. If initiatorName + is specified with iscsiInterface simultaneously, new iSCSI + interface : will be created + for the connection. + type: string + iqn: + description: Target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iSCSI Interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: iSCSI Target Portal List. The portal is either + an IP or ip_addr:port if the port is other than default + (typically TCP ports 860 and 3260). + items: + type: string + type: array + readOnly: + description: ReadOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: CHAP Secret for iSCSI target and initiator authentication + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + targetPortal: + description: iSCSI Target Portal. The Portal is either an + IP or ip_addr:port if the port is other than default (typically + TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'Volume''s name. Must be a DNS_LABEL and unique within + the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + nfs: + description: 'NFS represents an NFS mount on the host that shares + a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'Path that is exported by the NFS server. More + info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'ReadOnly here will force the NFS export to be + mounted with read-only permissions. Defaults to false. More + info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'Server is the hostname or IP address of the + NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'PersistentVolumeClaimVolumeSource represents a reference + to a PersistentVolumeClaim in the same namespace. More info: + https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + claimName: + description: 'ClaimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: PhotonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host machine + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: ID that identifies Photon Controller persistent + disk + type: string + required: + - pdID + type: object + portworxVolume: + description: PortworxVolume represents a portworx volume attached + and mounted on kubelets host machine + properties: + fsType: + description: FSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs". Implicitly inferred to be "ext4" + if unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly here + will force the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: VolumeID uniquely identifies a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: Items for all in one resources secrets, configmaps, + and downward API + properties: + defaultMode: + description: Mode bits to use on created files by default. + Must be a value between 0 and 0777. Directories within the + path are not affected by this setting. This might be in + conflict with other options that affect the file mode, like + fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: list of volume projections + items: + description: Projection that may be projected along with + other supported volume types + properties: + configMap: + description: information about the configMap data to + project + properties: + items: + description: If unspecified, each key-value pair + in the Data field of the referenced ConfigMap + will be projected into the volume as a file whose + name is the key and content is the value. If specified, + the listed keys will be projected into the specified + paths, and unlisted keys will not be present. + If a key is specified which is not present in + the ConfigMap, the volume setup will error unless + it is marked optional. Paths must be relative + and may not contain the '..' path or start with + '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to use on + this file, must be a value between 0 and + 0777. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the file + to map the key to. May not be an absolute + path. May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + keys must be defined + type: boolean + type: object + downwardAPI: + description: information about the downwardAPI data + to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing the + pod field + properties: + fieldRef: + description: 'Required: Selects a field of + the pod: only annotations, labels, name + and namespace are supported.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits to use on + this file, must be a value between 0 and + 0777. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must + not be absolute or contain the ''..'' path. + Must be utf-8 encoded. The first item of + the relative path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + description: Specifies the output format + of the exposed resources, defaults to + "1" + type: string + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + description: information about the secret data to project + properties: + items: + description: If unspecified, each key-value pair + in the Data field of the referenced Secret will + be projected into the volume as a file whose name + is the key and content is the value. If specified, + the listed keys will be projected into the specified + paths, and unlisted keys will not be present. + If a key is specified which is not present in + the Secret, the volume setup will error unless + it is marked optional. Paths must be relative + and may not contain the '..' path or start with + '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to use on + this file, must be a value between 0 and + 0777. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the file + to map the key to. May not be an absolute + path. May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + type: object + serviceAccountToken: + description: information about the serviceAccountToken + data to project + properties: + audience: + description: Audience is the intended audience of + the token. A recipient of a token must identify + itself with an identifier specified in the audience + of the token, and otherwise should reject the + token. The audience defaults to the identifier + of the apiserver. + type: string + expirationSeconds: + description: ExpirationSeconds is the requested + duration of validity of the service account token. + As the token approaches expiration, the kubelet + volume plugin will proactively rotate the service + account token. The kubelet will start trying to + rotate the token if the token is older than 80 + percent of its time to live or if the token is + older than 24 hours.Defaults to 1 hour and must + be at least 10 minutes. + format: int64 + type: integer + path: + description: Path is the path relative to the mount + point of the file to project the token into. + type: string + required: + - path + type: object + type: object + type: array + required: + - sources + type: object + quobyte: + description: Quobyte represents a Quobyte mount on the host that + shares a pod's lifetime + properties: + group: + description: Group to map volume access to Default is no group + type: string + readOnly: + description: ReadOnly here will force the Quobyte volume to + be mounted with read-only permissions. Defaults to false. + type: boolean + registry: + description: Registry represents a single or multiple Quobyte + Registry services specified as a string as host:port pair + (multiple entries are separated with commas) which acts + as the central registry for volumes + type: string + tenant: + description: Tenant owning the given Quobyte volume in the + Backend Used with dynamically provisioned Quobyte volumes, + value is set by the plugin + type: string + user: + description: User to map volume access to Defaults to serivceaccount + user + type: string + volume: + description: Volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'RBD represents a Rados Block Device mount on the + host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' + properties: + fsType: + description: 'Filesystem type of the volume that you want + to mount. Tip: Ensure that the filesystem type is supported + by the host operating system. Examples: "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem from compromising + the machine' + type: string + image: + description: 'The rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'Keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'A collection of Ceph monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'The rados pool name. Default is rbd. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'SecretRef is name of the authentication secret + for RBDUser. If provided overrides keyring. Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + user: + description: 'The rados user name. Default is admin. More + info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: ScaleIO represents a ScaleIO persistent volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Default is "xfs". + type: string + gateway: + description: The host address of the ScaleIO API Gateway. + type: string + protectionDomain: + description: The name of the ScaleIO Protection Domain for + the configured storage. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly here + will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef references to the secret for ScaleIO + user and other sensitive information. If this is not provided, + Login operation will fail. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + sslEnabled: + description: Flag to enable/disable SSL communication with + Gateway, default false + type: boolean + storageMode: + description: Indicates whether the storage for a volume should + be ThickProvisioned or ThinProvisioned. Default is ThinProvisioned. + type: string + storagePool: + description: The ScaleIO Storage Pool associated with the + protection domain. + type: string + system: + description: The name of the storage system as configured + in ScaleIO. + type: string + volumeName: + description: The name of a volume already created in the ScaleIO + system that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'Secret represents a secret that should populate + this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'Optional: mode bits to use on created files + by default. Must be a value between 0 and 0777. Defaults + to 0644. Directories within the path are not affected by + this setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair in the Data + field of the referenced Secret will be projected into the + volume as a file whose name is the key and content is the + value. If specified, the listed keys will be projected into + the specified paths, and unlisted keys will not be present. + If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. + Paths must be relative and may not contain the '..' path + or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to use on this file, + must be a value between 0 and 0777. If not specified, + the volume defaultMode will be used. This might be + in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode + bits set.' + format: int32 + type: integer + path: + description: The relative path of the file to map the + key to. May not be an absolute path. May not contain + the path element '..'. May not start with the string + '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: Specify whether the Secret or its keys must be + defined + type: boolean + secretName: + description: 'Name of the secret in the pod''s namespace to + use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: StorageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly here + will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef specifies the secret to use for obtaining + the StorageOS API credentials. If not specified, default + values will be attempted. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + volumeName: + description: VolumeName is the human-readable name of the + StorageOS volume. Volume names are only unique within a + namespace. + type: string + volumeNamespace: + description: VolumeNamespace specifies the scope of the volume + within StorageOS. If no namespace is specified then the + Pod's namespace will be used. This allows the Kubernetes + name scoping to be mirrored within StorageOS for tighter + integration. Set VolumeName to any name to override the + default behaviour. Set to "default" if you are not using + namespaces within StorageOS. Namespaces that do not pre-exist + within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: VsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: Storage Policy Based Management (SPBM) profile + ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: Storage Policy Based Management (SPBM) profile + name. + type: string + volumePath: + description: Path that identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + status: + description: 'Most recent observed status of the Alertmanager cluster. Read-only. + Not included when requesting from the apiserver, only from the Prometheus + Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' + properties: + availableReplicas: + description: Total number of available pods (ready for at least minReadySeconds) + targeted by this Alertmanager cluster. + format: int32 + type: integer + paused: + description: Represents whether any actions on the underlaying managed + objects are being performed. Only delete actions will be performed. + type: boolean + replicas: + description: Total number of non-terminated pods targeted by this Alertmanager + cluster (their labels match the selector). + format: int32 + type: integer + unavailableReplicas: + description: Total number of unavailable pods targeted by this Alertmanager + cluster. + format: int32 + type: integer + updatedReplicas: + description: Total number of non-terminated pods targeted by this Alertmanager + cluster that have the desired version spec. + format: int32 + type: integer + required: + - availableReplicas + - paused + - replicas + - unavailableReplicas + - updatedReplicas + type: object + required: + - spec + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true diff --git a/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-podmonitor.yaml b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-podmonitor.yaml new file mode 100755 index 000000000..9cf3c42e4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-podmonitor.yaml @@ -0,0 +1,260 @@ +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.38/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.4 + creationTimestamp: null + name: podmonitors.monitoring.coreos.com +spec: + group: monitoring.coreos.com + names: + kind: PodMonitor + listKind: PodMonitorList + plural: podmonitors + singular: podmonitor + preserveUnknownFields: false + scope: Namespaced + validation: + openAPIV3Schema: + description: PodMonitor defines monitoring for a set of pods. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Specification of desired Pod selection for target discovery + by Prometheus. + properties: + jobLabel: + description: The label to use to retrieve the job name from. + type: string + namespaceSelector: + description: Selector to select which namespaces the Endpoints objects + are discovered from. + properties: + any: + description: Boolean describing whether all namespaces are selected + in contrast to a list restricting them. + type: boolean + matchNames: + description: List of namespace names. + items: + type: string + type: array + type: object + podMetricsEndpoints: + description: A list of endpoints allowed as part of this PodMonitor. + items: + description: PodMetricsEndpoint defines a scrapeable endpoint of a + Kubernetes Pod serving Prometheus metrics. + properties: + honorLabels: + description: HonorLabels chooses the metric's labels on collisions + with target labels. + type: boolean + honorTimestamps: + description: HonorTimestamps controls whether Prometheus respects + the timestamps present in scraped data. + type: boolean + interval: + description: Interval at which metrics should be scraped + type: string + metricRelabelings: + description: MetricRelabelConfigs to apply to samples before ingestion. + items: + description: 'RelabelConfig allows dynamic rewriting of the + label set, being applied to samples before ingestion. It defines + ``-section of Prometheus configuration. + More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' + properties: + action: + description: Action to perform based on regex matching. + Default is 'replace' + type: string + modulus: + description: Modulus to take of the hash of the source label + values. + format: int64 + type: integer + regex: + description: Regular expression against which the extracted + value is matched. Default is '(.*)' + type: string + replacement: + description: Replacement value against which a regex replace + is performed if the regular expression matches. Regex + capture groups are available. Default is '$1' + type: string + separator: + description: Separator placed between concatenated source + label values. default is ';'. + type: string + sourceLabels: + description: The source labels select values from existing + labels. Their content is concatenated using the configured + separator and matched against the configured regular expression + for the replace, keep, and drop actions. + items: + type: string + type: array + targetLabel: + description: Label to which the resulting value is written + in a replace action. It is mandatory for replace actions. + Regex capture groups are available. + type: string + type: object + type: array + params: + additionalProperties: + items: + type: string + type: array + description: Optional HTTP URL parameters + type: object + path: + description: HTTP path to scrape for metrics. + type: string + port: + description: Name of the pod port this endpoint refers to. Mutually + exclusive with targetPort. + type: string + proxyUrl: + description: ProxyURL eg http://proxyserver:2195 Directs scrapes + to proxy through this endpoint. + type: string + relabelings: + description: 'RelabelConfigs to apply to samples before ingestion. + More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config' + items: + description: 'RelabelConfig allows dynamic rewriting of the + label set, being applied to samples before ingestion. It defines + ``-section of Prometheus configuration. + More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' + properties: + action: + description: Action to perform based on regex matching. + Default is 'replace' + type: string + modulus: + description: Modulus to take of the hash of the source label + values. + format: int64 + type: integer + regex: + description: Regular expression against which the extracted + value is matched. Default is '(.*)' + type: string + replacement: + description: Replacement value against which a regex replace + is performed if the regular expression matches. Regex + capture groups are available. Default is '$1' + type: string + separator: + description: Separator placed between concatenated source + label values. default is ';'. + type: string + sourceLabels: + description: The source labels select values from existing + labels. Their content is concatenated using the configured + separator and matched against the configured regular expression + for the replace, keep, and drop actions. + items: + type: string + type: array + targetLabel: + description: Label to which the resulting value is written + in a replace action. It is mandatory for replace actions. + Regex capture groups are available. + type: string + type: object + type: array + scheme: + description: HTTP scheme to use for scraping. + type: string + scrapeTimeout: + description: Timeout after which the scrape is ended + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: 'Deprecated: Use ''port'' instead.' + x-kubernetes-int-or-string: true + type: object + type: array + podTargetLabels: + description: PodTargetLabels transfers labels on the Kubernetes Pod + onto the target. + items: + type: string + type: array + sampleLimit: + description: SampleLimit defines per-scrape limit on number of scraped + samples that will be accepted. + format: int64 + type: integer + selector: + description: Selector to select Pod objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains + values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator + is In or NotIn, the values array must be non-empty. If the + operator is Exists or DoesNotExist, the values array must + be empty. This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator is + "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + required: + - podMetricsEndpoints + - selector + type: object + required: + - spec + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true diff --git a/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-prometheus.yaml b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-prometheus.yaml new file mode 100755 index 000000000..704379fb2 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-prometheus.yaml @@ -0,0 +1,6002 @@ +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.38/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.4 + creationTimestamp: null + name: prometheuses.monitoring.coreos.com +spec: + additionalPrinterColumns: + - JSONPath: .spec.version + description: The version of Prometheus + name: Version + type: string + - JSONPath: .spec.replicas + description: The desired replicas number of Prometheuses + name: Replicas + type: integer + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + group: monitoring.coreos.com + names: + kind: Prometheus + listKind: PrometheusList + plural: prometheuses + singular: prometheus + preserveUnknownFields: false + scope: Namespaced + subresources: {} + validation: + openAPIV3Schema: + description: Prometheus defines a Prometheus deployment. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: 'Specification of the desired behavior of the Prometheus cluster. + More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' + properties: + additionalAlertManagerConfigs: + description: 'AdditionalAlertManagerConfigs allows specifying a key + of a Secret containing additional Prometheus AlertManager configurations. + AlertManager configurations specified are appended to the configurations + generated by the Prometheus Operator. Job configurations specified + must have the form as specified in the official Prometheus documentation: + https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alertmanager_config. + As AlertManager configs are appended, the user is responsible to make + sure it is valid. Note that using this feature may expose the possibility + to break upgrades of Prometheus. It is advised to review Prometheus + release notes to ensure that no incompatible AlertManager configs + are going to break Prometheus after the upgrade.' + properties: + key: + description: The key of the secret to select from. Must be a valid + secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + additionalAlertRelabelConfigs: + description: 'AdditionalAlertRelabelConfigs allows specifying a key + of a Secret containing additional Prometheus alert relabel configurations. + Alert relabel configurations specified are appended to the configurations + generated by the Prometheus Operator. Alert relabel configurations + specified must have the form as specified in the official Prometheus + documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs. + As alert relabel configs are appended, the user is responsible to + make sure it is valid. Note that using this feature may expose the + possibility to break upgrades of Prometheus. It is advised to review + Prometheus release notes to ensure that no incompatible alert relabel + configs are going to break Prometheus after the upgrade.' + properties: + key: + description: The key of the secret to select from. Must be a valid + secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + additionalScrapeConfigs: + description: 'AdditionalScrapeConfigs allows specifying a key of a Secret + containing additional Prometheus scrape configurations. Scrape configurations + specified are appended to the configurations generated by the Prometheus + Operator. Job configurations specified must have the form as specified + in the official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. + As scrape configs are appended, the user is responsible to make sure + it is valid. Note that using this feature may expose the possibility + to break upgrades of Prometheus. It is advised to review Prometheus + release notes to ensure that no incompatible scrape configs are going + to break Prometheus after the upgrade.' + properties: + key: + description: The key of the secret to select from. Must be a valid + secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + affinity: + description: If specified, the pod's scheduling constraints. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes + that satisfy the affinity expressions specified by this field, + but it may choose a node that violates one or more of the + expressions. The node that is most preferred is the one with + the greatest sum of weights, i.e. for each node that meets + all of the scheduling requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating through + the elements of this field and adding "weight" to the sum + if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: An empty preferred scheduling term matches all + objects with implicit weight 0 (i.e. it's a no-op). A null + preferred scheduling term matches no objects (i.e. is also + a no-op). + properties: + preference: + description: A node selector term, associated with the + corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the + operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be + empty. If the operator is Gt or Lt, the values + array must have a single element, which will + be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the + operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be + empty. If the operator is Gt or Lt, the values + array must have a single element, which will + be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by this + field are not met at scheduling time, the pod will not be + scheduled onto the node. If the affinity requirements specified + by this field cease to be met at some point during pod execution + (e.g. due to an update), the system may or may not try to + eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The + terms are ORed. + items: + description: A null or empty node selector term matches + no objects. The requirements of them are ANDed. The + TopologySelectorTerm type implements a subset of the + NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the + operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be + empty. If the operator is Gt or Lt, the values + array must have a single element, which will + be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the + operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be + empty. If the operator is Gt or Lt, the values + array must have a single element, which will + be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes + that satisfy the affinity expressions specified by this field, + but it may choose a node that violates one or more of the + expressions. The node that is most preferred is the one with + the greatest sum of weights, i.e. for each node that meets + all of the scheduling requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating through + the elements of this field and adding "weight" to the sum + if the node has pods which matches the corresponding podAffinityTerm; + the node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement is + a selector that contains values, a key, and + an operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. This array + is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified namespaces, + where co-located is defined as running on a node + whose value of the label with key topologyKey matches + that of any node on which any of the selected pods + is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the corresponding + podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by this + field are not met at scheduling time, the pod will not be + scheduled onto the node. If the affinity requirements specified + by this field cease to be met at some point during pod execution + (e.g. due to a pod label update), the system may or may not + try to eventually evict the pod from its node. When there + are multiple elements, the lists of nodes corresponding to + each podAffinityTerm are intersected, i.e. all terms must + be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) that + this pod should be co-located (affinity) or not co-located + (anti-affinity) with, where co-located is defined as running + on a node whose value of the label with key + matches that of any node on which a pod of the set of pods + is running + properties: + labelSelector: + description: A label query over a set of resources, in + this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values + array must be non-empty. If the operator is + Exists or DoesNotExist, the values array must + be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces the + labelSelector applies to (matches against); null or + empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where + co-located is defined as running on a node whose value + of the label with key topologyKey matches that of any + node on which any of the selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some other + pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes + that satisfy the anti-affinity expressions specified by this + field, but it may choose a node that violates one or more + of the expressions. The node that is most preferred is the + one with the greatest sum of weights, i.e. for each node that + meets all of the scheduling requirements (resource request, + requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field + and adding "weight" to the sum if the node has pods which + matches the corresponding podAffinityTerm; the node(s) with + the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement is + a selector that contains values, a key, and + an operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. This array + is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified namespaces, + where co-located is defined as running on a node + whose value of the label with key topologyKey matches + that of any node on which any of the selected pods + is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the corresponding + podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements specified by + this field are not met at scheduling time, the pod will not + be scheduled onto the node. If the anti-affinity requirements + specified by this field cease to be met at some point during + pod execution (e.g. due to a pod label update), the system + may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding + to each podAffinityTerm are intersected, i.e. all terms must + be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) that + this pod should be co-located (affinity) or not co-located + (anti-affinity) with, where co-located is defined as running + on a node whose value of the label with key + matches that of any node on which a pod of the set of pods + is running + properties: + labelSelector: + description: A label query over a set of resources, in + this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values + array must be non-empty. If the operator is + Exists or DoesNotExist, the values array must + be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces the + labelSelector applies to (matches against); null or + empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where + co-located is defined as running on a node whose value + of the label with key topologyKey matches that of any + node on which any of the selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + alerting: + description: Define details regarding alerting. + properties: + alertmanagers: + description: AlertmanagerEndpoints Prometheus should fire alerts + against. + items: + description: AlertmanagerEndpoints defines a selection of a single + Endpoints object containing alertmanager IPs to fire alerts + against. + properties: + apiVersion: + description: Version of the Alertmanager API that Prometheus + uses to send alerts. It can be "v1" or "v2". + type: string + bearerTokenFile: + description: BearerTokenFile to read from filesystem to use + when authenticating to Alertmanager. + type: string + name: + description: Name of Endpoints object in Namespace. + type: string + namespace: + description: Namespace of Endpoints object. + type: string + pathPrefix: + description: Prefix for the HTTP path alerts are pushed to. + type: string + port: + anyOf: + - type: integer + - type: string + description: Port the Alertmanager API is exposed on. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use when firing alerts. + type: string + tlsConfig: + description: TLS Config to use for alertmanager connection. + properties: + ca: + description: Stuct containing the CA cert to use for the + targets. + properties: + configMap: + description: ConfigMap containing data to use for + the targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the + targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object + caFile: + description: Path to the CA cert in the Prometheus container + to use for the targets. + type: string + cert: + description: Struct containing the client cert file for + the targets. + properties: + configMap: + description: ConfigMap containing data to use for + the targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the + targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object + certFile: + description: Path to the client cert file in the Prometheus + container for the targets. + type: string + insecureSkipVerify: + description: Disable target certificate validation. + type: boolean + keyFile: + description: Path to the client key file in the Prometheus + container for the targets. + type: string + keySecret: + description: Secret containing the client key file for + the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + serverName: + description: Used to verify the hostname for the targets. + type: string + type: object + required: + - name + - namespace + - port + type: object + type: array + required: + - alertmanagers + type: object + apiserverConfig: + description: APIServerConfig allows specifying a host and auth methods + to access apiserver. If left empty, Prometheus is assumed to run inside + of the cluster and will discover API servers automatically and use + the pod's CA certificate and bearer token file at /var/run/secrets/kubernetes.io/serviceaccount/. + properties: + basicAuth: + description: BasicAuth allow an endpoint to authenticate over basic + authentication + properties: + password: + description: The secret in the service monitor namespace that + contains the password for authentication. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + username: + description: The secret in the service monitor namespace that + contains the username for authentication. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + type: object + bearerToken: + description: Bearer token for accessing apiserver. + type: string + bearerTokenFile: + description: File to read bearer token for accessing apiserver. + type: string + host: + description: Host of apiserver. A valid string consisting of a hostname + or IP followed by an optional port number + type: string + tlsConfig: + description: TLS Config to use for accessing apiserver. + properties: + ca: + description: Stuct containing the CA cert to use for the targets. + properties: + configMap: + description: ConfigMap containing data to use for the targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + type: object + caFile: + description: Path to the CA cert in the Prometheus container + to use for the targets. + type: string + cert: + description: Struct containing the client cert file for the + targets. + properties: + configMap: + description: ConfigMap containing data to use for the targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + type: object + certFile: + description: Path to the client cert file in the Prometheus + container for the targets. + type: string + insecureSkipVerify: + description: Disable target certificate validation. + type: boolean + keyFile: + description: Path to the client key file in the Prometheus container + for the targets. + type: string + keySecret: + description: Secret containing the client key file for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + serverName: + description: Used to verify the hostname for the targets. + type: string + type: object + required: + - host + type: object + arbitraryFSAccessThroughSMs: + description: ArbitraryFSAccessThroughSMs configures whether configuration + based on a service monitor can access arbitrary files on the file + system of the Prometheus container e.g. bearer token files. + properties: + deny: + type: boolean + type: object + baseImage: + description: Base image to use for a Prometheus deployment. + type: string + configMaps: + description: ConfigMaps is a list of ConfigMaps in the same namespace + as the Prometheus object, which shall be mounted into the Prometheus + Pods. The ConfigMaps are mounted into /etc/prometheus/configmaps/. + items: + type: string + type: array + containers: + description: 'Containers allows injecting additional containers or modifying + operator generated containers. This can be used to allow adding an + authentication proxy to a Prometheus pod or to change the behavior + of an operator generated container. Containers described here modify + an operator generated container if they share the same name and modifications + are done via a strategic merge patch. The current container names + are: `prometheus`, `prometheus-config-reloader`, `rules-configmap-reloader`, + and `thanos-sidecar`. Overriding containers is entirely outside the + scope of what the maintainers will support and by doing so, you accept + that this behaviour may break at any time without notice.' + items: + description: A single application container that you want to run within + a pod. + properties: + args: + description: 'Arguments to the entrypoint. The docker image''s + CMD is used if this is not provided. Variable references $(VAR_NAME) + are expanded using the container''s environment. If a variable + cannot be resolved, the reference in the input string will be + unchanged. The $(VAR_NAME) syntax can be escaped with a double + $$, ie: $$(VAR_NAME). Escaped references will never be expanded, + regardless of whether the variable exists or not. Cannot be + updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within a shell. The + docker image''s ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, the reference + in the input string will be unchanged. The $(VAR_NAME) syntax + can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable exists + or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previous defined environment variables in the + container and any service environment variables. If a + variable cannot be resolved, the reference in the input + string will be unchanged. The $(VAR_NAME) syntax can be + escaped with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable + exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, metadata.labels, metadata.annotations, + spec.nodeName, spec.serviceAccountName, status.hostIP, + status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + description: Specifies the output format of the + exposed resources, defaults to "1" + type: string + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment variables + in the container. The keys defined within a source must be a + C_IDENTIFIER. All invalid keys will be reported as an event + when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take + precedence. Values defined by an Env with a duplicate key will + take precedence. Cannot be updated. + items: + description: EnvFromSource represents the source of a set of + ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + prefix: + description: An optional identifier to prepend to each key + in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + type: object + type: array + image: + description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management + to default or override container images in workload controllers + like Deployments and StatefulSets.' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent + otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Actions that the management system should take in + response to container lifecycle events. Cannot be updated. + properties: + postStart: + description: 'PostStart is called immediately after a container + is created. If the handler fails, the container is terminated + and restarted according to its restart policy. Other management + of the container blocks until the hook completes. More info: + https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's filesystem. + The command is simply exec'd, it is not run inside + a shell, so traditional shell instructions ('|', + etc) won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is treated + as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately before a container + is terminated due to an API request or management event + such as liveness/startup probe failure, preemption, resource + contention, etc. The handler is not called if the container + crashes or exits. The reason for termination is passed to + the handler. The Pod''s termination grace period countdown + begins before the PreStop hooked is executed. Regardless + of the outcome of the handler, the container will eventually + terminate within the Pod''s termination grace period. Other + management of the container blocks until the hook completes + or until the termination grace period is reached. More info: + https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's filesystem. + The command is simply exec'd, it is not run inside + a shell, so traditional shell instructions ('|', + etc) won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is treated + as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: 'Periodic probe of container liveness. Container + will be restarted if the probe fails. Cannot be updated. More + info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + name: + description: Name of the container specified as a DNS_LABEL. Each + container in a pod must have a unique name (DNS_LABEL). Cannot + be updated. + type: string + ports: + description: List of ports to expose from the container. Exposing + a port here gives the system additional information about the + network connections a container uses, but is primarily informational. + Not specifying a port here DOES NOT prevent that port from being + exposed. Any port which is listening on the default "0.0.0.0" + address inside a container will be accessible from the network. + Cannot be updated. + items: + description: ContainerPort represents a network port in a single + container. + properties: + containerPort: + description: Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: Number of port to expose on the host. If specified, + this must be a valid port number, 0 < x < 65536. If HostNetwork + is specified, this must match ContainerPort. Most containers + do not need this. + format: int32 + type: integer + name: + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port in a pod must + have a unique name. Name for the port that can be referred + to by services. + type: string + protocol: + description: Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + readinessProbe: + description: 'Periodic probe of container service readiness. Container + will be removed from service endpoints if the probe fails. Cannot + be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + resources: + description: 'Compute Resources required by this container. Cannot + be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + properties: + limits: + additionalProperties: + type: string + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + type: string + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + securityContext: + description: 'Security options the pod should run with. More info: + https://kubernetes.io/docs/concepts/policy/security-context/ + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether a + process can gain more privileges than its parent process. + This bool directly controls if the no_new_privs flag will + be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as Privileged + 2) has CAP_SYS_ADMIN' + type: boolean + capabilities: + description: The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the + container runtime. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes in + privileged containers are essentially equivalent to root + on the host. Defaults to false. + type: boolean + procMount: + description: procMount denotes the type of proc mount to use + for the containers. The default is DefaultProcMount which + uses the container runtime defaults for readonly paths and + masked paths. This requires the ProcMountType feature flag + to be enabled. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only root filesystem. + Default is false. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container + process. Uses runtime default if unset. May also be set + in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a non-root + user. If true, the Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 (root) and fail + to start the container if it does. If unset or false, no + such validation will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container + process. Defaults to user specified in image metadata if + unspecified. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random + SELinux context for each container. May also be set in + PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + windowsOptions: + description: The Windows specific settings applied to all + containers. If unspecified, the options from the PodSecurityContext + will be used. If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission + webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec named + by the GMSACredentialSpecName field. This field is alpha-level + and is only honored by servers that enable the WindowsGMSA + feature flag. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. This field is alpha-level + and is only honored by servers that enable the WindowsGMSA + feature flag. + type: string + runAsUserName: + description: The UserName in Windows to run the entrypoint + of the container process. Defaults to the user specified + in image metadata if unspecified. May also be set in + PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. This field is beta-level and may be + disabled with the WindowsRunAsUserName feature flag. + type: string + type: object + type: object + startupProbe: + description: 'StartupProbe indicates that the Pod has successfully + initialized. If specified, no other probes are executed until + this completes successfully. If this probe fails, the Pod will + be restarted, just as if the livenessProbe failed. This can + be used to provide different probe parameters at the beginning + of a Pod''s lifecycle, when it might take a long time to load + data or warm a cache, than during steady-state operation. This + cannot be updated. This is an alpha feature enabled by the StartupProbe + feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + stdin: + description: Whether this container should allocate a buffer for + stdin in the container runtime. If this is not set, reads from + stdin in the container will always result in EOF. Default is + false. + type: boolean + stdinOnce: + description: Whether the container runtime should close the stdin + channel after it has been opened by a single attach. When stdin + is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container + start, is empty until the first client attaches to stdin, and + then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container + is restarted. If this flag is false, a container processes that + reads from stdin will never receive an EOF. Default is false + type: boolean + terminationMessagePath: + description: 'Optional: Path at which the file to which the container''s + termination message will be written is mounted into the container''s + filesystem. Message written is intended to be brief final status, + such as an assertion failure message. Will be truncated by the + node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. Defaults to /dev/termination-log. + Cannot be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message should be populated. + File will use the contents of terminationMessagePath to populate + the container status message on both success and failure. FallbackToLogsOnError + will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever + is smaller. Defaults to File. Cannot be updated. + type: string + tty: + description: Whether this container should allocate a TTY for + itself, also requires 'stdin' to be true. Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be + used by the container. This is a beta feature. + items: + description: volumeDevice describes a mapping of a raw block + device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container + that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: Path within the container at which the volume + should be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts are + propagated from the host to container and the other way + around. When not set, MountPropagationNone is used. This + field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the container's + volume should be mounted. Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from which + the container's volume should be mounted. Behaves similarly + to SubPath but environment variable references $(VAR_NAME) + are expanded using the container's environment. Defaults + to "" (volume's root). SubPathExpr and SubPath are mutually + exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If not specified, + the container runtime's default will be used, which might be + configured in the container image. Cannot be updated. + type: string + required: + - name + type: object + type: array + disableCompaction: + description: Disable prometheus compaction. + type: boolean + enableAdminAPI: + description: 'Enable access to prometheus web admin API. Defaults to + the value of `false`. WARNING: Enabling the admin APIs enables mutating + endpoints, to delete data, shutdown Prometheus, and more. Enabling + this should be done with care and the user is advised to add additional + authentication authorization via a proxy to ensure only clients authorized + to perform these actions can do so. For more information see https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis' + type: boolean + enforcedNamespaceLabel: + description: EnforcedNamespaceLabel enforces adding a namespace label + of origin for each alert and metric that is user created. The label + value will always be the namespace of the object that is being created. + type: string + evaluationInterval: + description: Interval between consecutive evaluations. + type: string + externalLabels: + additionalProperties: + type: string + description: The labels to add to any time series or alerts when communicating + with external systems (federation, remote storage, Alertmanager). + type: object + externalUrl: + description: The external URL the Prometheus instances will be available + under. This is necessary to generate correct URLs. This is necessary + if Prometheus is not served from root of a DNS name. + type: string + ignoreNamespaceSelectors: + description: IgnoreNamespaceSelectors if set to true will ignore NamespaceSelector + settings from the podmonitor and servicemonitor configs, and they + will only discover endpoints within their current namespace. Defaults + to false. + type: boolean + image: + description: Image if specified has precedence over baseImage, tag and + sha combinations. Specifying the version is still necessary to ensure + the Prometheus Operator knows what version of Prometheus is being + configured. + type: string + imagePullSecrets: + description: An optional list of references to secrets in the same namespace + to use for pulling prometheus and alertmanager images from registries + see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod + items: + description: LocalObjectReference contains enough information to let + you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + type: array + initContainers: + description: 'InitContainers allows adding initContainers to the pod + definition. Those can be used to e.g. fetch secrets for injection + into the Prometheus configuration from external sources. Any errors + during the execution of an initContainer will lead to a restart of + the Pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + Using initContainers for any use case other then secret fetching is + entirely outside the scope of what the maintainers will support and + by doing so, you accept that this behaviour may break at any time + without notice.' + items: + description: A single application container that you want to run within + a pod. + properties: + args: + description: 'Arguments to the entrypoint. The docker image''s + CMD is used if this is not provided. Variable references $(VAR_NAME) + are expanded using the container''s environment. If a variable + cannot be resolved, the reference in the input string will be + unchanged. The $(VAR_NAME) syntax can be escaped with a double + $$, ie: $$(VAR_NAME). Escaped references will never be expanded, + regardless of whether the variable exists or not. Cannot be + updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within a shell. The + docker image''s ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, the reference + in the input string will be unchanged. The $(VAR_NAME) syntax + can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable exists + or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previous defined environment variables in the + container and any service environment variables. If a + variable cannot be resolved, the reference in the input + string will be unchanged. The $(VAR_NAME) syntax can be + escaped with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable + exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, metadata.labels, metadata.annotations, + spec.nodeName, spec.serviceAccountName, status.hostIP, + status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + description: Specifies the output format of the + exposed resources, defaults to "1" + type: string + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment variables + in the container. The keys defined within a source must be a + C_IDENTIFIER. All invalid keys will be reported as an event + when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take + precedence. Values defined by an Env with a duplicate key will + take precedence. Cannot be updated. + items: + description: EnvFromSource represents the source of a set of + ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + prefix: + description: An optional identifier to prepend to each key + in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + type: object + type: array + image: + description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management + to default or override container images in workload controllers + like Deployments and StatefulSets.' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent + otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Actions that the management system should take in + response to container lifecycle events. Cannot be updated. + properties: + postStart: + description: 'PostStart is called immediately after a container + is created. If the handler fails, the container is terminated + and restarted according to its restart policy. Other management + of the container blocks until the hook completes. More info: + https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's filesystem. + The command is simply exec'd, it is not run inside + a shell, so traditional shell instructions ('|', + etc) won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is treated + as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately before a container + is terminated due to an API request or management event + such as liveness/startup probe failure, preemption, resource + contention, etc. The handler is not called if the container + crashes or exits. The reason for termination is passed to + the handler. The Pod''s termination grace period countdown + begins before the PreStop hooked is executed. Regardless + of the outcome of the handler, the container will eventually + terminate within the Pod''s termination grace period. Other + management of the container blocks until the hook completes + or until the termination grace period is reached. More info: + https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's filesystem. + The command is simply exec'd, it is not run inside + a shell, so traditional shell instructions ('|', + etc) won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is treated + as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: 'Periodic probe of container liveness. Container + will be restarted if the probe fails. Cannot be updated. More + info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + name: + description: Name of the container specified as a DNS_LABEL. Each + container in a pod must have a unique name (DNS_LABEL). Cannot + be updated. + type: string + ports: + description: List of ports to expose from the container. Exposing + a port here gives the system additional information about the + network connections a container uses, but is primarily informational. + Not specifying a port here DOES NOT prevent that port from being + exposed. Any port which is listening on the default "0.0.0.0" + address inside a container will be accessible from the network. + Cannot be updated. + items: + description: ContainerPort represents a network port in a single + container. + properties: + containerPort: + description: Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: Number of port to expose on the host. If specified, + this must be a valid port number, 0 < x < 65536. If HostNetwork + is specified, this must match ContainerPort. Most containers + do not need this. + format: int32 + type: integer + name: + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port in a pod must + have a unique name. Name for the port that can be referred + to by services. + type: string + protocol: + description: Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + readinessProbe: + description: 'Periodic probe of container service readiness. Container + will be removed from service endpoints if the probe fails. Cannot + be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + resources: + description: 'Compute Resources required by this container. Cannot + be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + properties: + limits: + additionalProperties: + type: string + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + type: string + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + securityContext: + description: 'Security options the pod should run with. More info: + https://kubernetes.io/docs/concepts/policy/security-context/ + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether a + process can gain more privileges than its parent process. + This bool directly controls if the no_new_privs flag will + be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as Privileged + 2) has CAP_SYS_ADMIN' + type: boolean + capabilities: + description: The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the + container runtime. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes in + privileged containers are essentially equivalent to root + on the host. Defaults to false. + type: boolean + procMount: + description: procMount denotes the type of proc mount to use + for the containers. The default is DefaultProcMount which + uses the container runtime defaults for readonly paths and + masked paths. This requires the ProcMountType feature flag + to be enabled. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only root filesystem. + Default is false. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container + process. Uses runtime default if unset. May also be set + in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a non-root + user. If true, the Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 (root) and fail + to start the container if it does. If unset or false, no + such validation will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container + process. Defaults to user specified in image metadata if + unspecified. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random + SELinux context for each container. May also be set in + PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + windowsOptions: + description: The Windows specific settings applied to all + containers. If unspecified, the options from the PodSecurityContext + will be used. If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission + webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec named + by the GMSACredentialSpecName field. This field is alpha-level + and is only honored by servers that enable the WindowsGMSA + feature flag. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. This field is alpha-level + and is only honored by servers that enable the WindowsGMSA + feature flag. + type: string + runAsUserName: + description: The UserName in Windows to run the entrypoint + of the container process. Defaults to the user specified + in image metadata if unspecified. May also be set in + PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. This field is beta-level and may be + disabled with the WindowsRunAsUserName feature flag. + type: string + type: object + type: object + startupProbe: + description: 'StartupProbe indicates that the Pod has successfully + initialized. If specified, no other probes are executed until + this completes successfully. If this probe fails, the Pod will + be restarted, just as if the livenessProbe failed. This can + be used to provide different probe parameters at the beginning + of a Pod''s lifecycle, when it might take a long time to load + data or warm a cache, than during steady-state operation. This + cannot be updated. This is an alpha feature enabled by the StartupProbe + feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + stdin: + description: Whether this container should allocate a buffer for + stdin in the container runtime. If this is not set, reads from + stdin in the container will always result in EOF. Default is + false. + type: boolean + stdinOnce: + description: Whether the container runtime should close the stdin + channel after it has been opened by a single attach. When stdin + is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container + start, is empty until the first client attaches to stdin, and + then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container + is restarted. If this flag is false, a container processes that + reads from stdin will never receive an EOF. Default is false + type: boolean + terminationMessagePath: + description: 'Optional: Path at which the file to which the container''s + termination message will be written is mounted into the container''s + filesystem. Message written is intended to be brief final status, + such as an assertion failure message. Will be truncated by the + node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. Defaults to /dev/termination-log. + Cannot be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message should be populated. + File will use the contents of terminationMessagePath to populate + the container status message on both success and failure. FallbackToLogsOnError + will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever + is smaller. Defaults to File. Cannot be updated. + type: string + tty: + description: Whether this container should allocate a TTY for + itself, also requires 'stdin' to be true. Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be + used by the container. This is a beta feature. + items: + description: volumeDevice describes a mapping of a raw block + device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container + that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: Path within the container at which the volume + should be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts are + propagated from the host to container and the other way + around. When not set, MountPropagationNone is used. This + field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the container's + volume should be mounted. Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from which + the container's volume should be mounted. Behaves similarly + to SubPath but environment variable references $(VAR_NAME) + are expanded using the container's environment. Defaults + to "" (volume's root). SubPathExpr and SubPath are mutually + exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If not specified, + the container runtime's default will be used, which might be + configured in the container image. Cannot be updated. + type: string + required: + - name + type: object + type: array + listenLocal: + description: ListenLocal makes the Prometheus server listen on loopback, + so that it does not bind against the Pod IP. + type: boolean + logFormat: + description: Log format for Prometheus to be configured with. + type: string + logLevel: + description: Log level for Prometheus to be configured with. + type: string + nodeSelector: + additionalProperties: + type: string + description: Define which Nodes the Pods are scheduled on. + type: object + overrideHonorLabels: + description: OverrideHonorLabels if set to true overrides all user configured + honor_labels. If HonorLabels is set in ServiceMonitor or PodMonitor + to true, this overrides honor_labels to false. + type: boolean + overrideHonorTimestamps: + description: OverrideHonorTimestamps allows to globally enforce honoring + timestamps in all scrape configs. + type: boolean + paused: + description: When a Prometheus deployment is paused, no actions except + for deletion will be performed on the underlying objects. + type: boolean + podMetadata: + description: PodMetadata configures Labels and Annotations which are + propagated to the prometheus pods. + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value map stored + with a resource that may be set by external tools to store and + retrieve arbitrary metadata. They are not queryable and should + be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations' + type: object + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can be used to + organize and categorize (scope and select) objects. May match + selectors of replication controllers and services. More info: + http://kubernetes.io/docs/user-guide/labels' + type: object + type: object + podMonitorNamespaceSelector: + description: Namespaces to be selected for PodMonitor discovery. If + nil, only check own namespace. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains + values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator + is In or NotIn, the values array must be non-empty. If the + operator is Exists or DoesNotExist, the values array must + be empty. This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator is + "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + podMonitorSelector: + description: '*Experimental* PodMonitors to be selected for target discovery.' + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains + values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator + is In or NotIn, the values array must be non-empty. If the + operator is Exists or DoesNotExist, the values array must + be empty. This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator is + "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + portName: + description: Port name used for the pods and governing service. This + defaults to web + type: string + priorityClassName: + description: Priority class assigned to the Pods + type: string + prometheusExternalLabelName: + description: Name of Prometheus external label used to denote Prometheus + instance name. Defaults to the value of `prometheus`. External label + will _not_ be added when value is set to empty string (`""`). + type: string + query: + description: QuerySpec defines the query command line flags when starting + Prometheus. + properties: + lookbackDelta: + description: The delta difference allowed for retrieving metrics + during expression evaluations. + type: string + maxConcurrency: + description: Number of concurrent queries that can be run at once. + format: int32 + type: integer + maxSamples: + description: Maximum number of samples a single query can load into + memory. Note that queries will fail if they would load more samples + than this into memory, so this also limits the number of samples + a query can return. + format: int32 + type: integer + timeout: + description: Maximum time a query may take before being aborted. + type: string + type: object + remoteRead: + description: If specified, the remote_read spec. This is an experimental + feature, it may change in any upcoming release in a breaking way. + items: + description: RemoteReadSpec defines the remote_read configuration + for prometheus. + properties: + basicAuth: + description: BasicAuth for the URL. + properties: + password: + description: The secret in the service monitor namespace that + contains the password for authentication. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + username: + description: The secret in the service monitor namespace that + contains the username for authentication. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + type: object + bearerToken: + description: bearer token for remote read. + type: string + bearerTokenFile: + description: File to read bearer token for remote read. + type: string + proxyUrl: + description: Optional ProxyURL + type: string + readRecent: + description: Whether reads should be made for queries for time + ranges that the local storage should have complete data for. + type: boolean + remoteTimeout: + description: Timeout for requests to the remote read endpoint. + type: string + requiredMatchers: + additionalProperties: + type: string + description: An optional list of equality matchers which have + to be present in a selector to query the remote read endpoint. + type: object + tlsConfig: + description: TLS Config to use for remote read. + properties: + ca: + description: Stuct containing the CA cert to use for the targets. + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + caFile: + description: Path to the CA cert in the Prometheus container + to use for the targets. + type: string + cert: + description: Struct containing the client cert file for the + targets. + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + certFile: + description: Path to the client cert file in the Prometheus + container for the targets. + type: string + insecureSkipVerify: + description: Disable target certificate validation. + type: boolean + keyFile: + description: Path to the client key file in the Prometheus + container for the targets. + type: string + keySecret: + description: Secret containing the client key file for the + targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + serverName: + description: Used to verify the hostname for the targets. + type: string + type: object + url: + description: The URL of the endpoint to send samples to. + type: string + required: + - url + type: object + type: array + remoteWrite: + description: If specified, the remote_write spec. This is an experimental + feature, it may change in any upcoming release in a breaking way. + items: + description: RemoteWriteSpec defines the remote_write configuration + for prometheus. + properties: + basicAuth: + description: BasicAuth for the URL. + properties: + password: + description: The secret in the service monitor namespace that + contains the password for authentication. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + username: + description: The secret in the service monitor namespace that + contains the username for authentication. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + type: object + bearerToken: + description: File to read bearer token for remote write. + type: string + bearerTokenFile: + description: File to read bearer token for remote write. + type: string + proxyUrl: + description: Optional ProxyURL + type: string + queueConfig: + description: QueueConfig allows tuning of the remote write queue + parameters. + properties: + batchSendDeadline: + description: BatchSendDeadline is the maximum time a sample + will wait in buffer. + type: string + capacity: + description: Capacity is the number of samples to buffer per + shard before we start dropping them. + type: integer + maxBackoff: + description: MaxBackoff is the maximum retry delay. + type: string + maxRetries: + description: MaxRetries is the maximum number of times to + retry a batch on recoverable errors. + type: integer + maxSamplesPerSend: + description: MaxSamplesPerSend is the maximum number of samples + per send. + type: integer + maxShards: + description: MaxShards is the maximum number of shards, i.e. + amount of concurrency. + type: integer + minBackoff: + description: MinBackoff is the initial retry delay. Gets doubled + for every retry. + type: string + minShards: + description: MinShards is the minimum number of shards, i.e. + amount of concurrency. + type: integer + type: object + remoteTimeout: + description: Timeout for requests to the remote write endpoint. + type: string + tlsConfig: + description: TLS Config to use for remote write. + properties: + ca: + description: Stuct containing the CA cert to use for the targets. + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + caFile: + description: Path to the CA cert in the Prometheus container + to use for the targets. + type: string + cert: + description: Struct containing the client cert file for the + targets. + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + certFile: + description: Path to the client cert file in the Prometheus + container for the targets. + type: string + insecureSkipVerify: + description: Disable target certificate validation. + type: boolean + keyFile: + description: Path to the client key file in the Prometheus + container for the targets. + type: string + keySecret: + description: Secret containing the client key file for the + targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + serverName: + description: Used to verify the hostname for the targets. + type: string + type: object + url: + description: The URL of the endpoint to send samples to. + type: string + writeRelabelConfigs: + description: The list of remote write relabel configurations. + items: + description: 'RelabelConfig allows dynamic rewriting of the + label set, being applied to samples before ingestion. It defines + ``-section of Prometheus configuration. + More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' + properties: + action: + description: Action to perform based on regex matching. + Default is 'replace' + type: string + modulus: + description: Modulus to take of the hash of the source label + values. + format: int64 + type: integer + regex: + description: Regular expression against which the extracted + value is matched. Default is '(.*)' + type: string + replacement: + description: Replacement value against which a regex replace + is performed if the regular expression matches. Regex + capture groups are available. Default is '$1' + type: string + separator: + description: Separator placed between concatenated source + label values. default is ';'. + type: string + sourceLabels: + description: The source labels select values from existing + labels. Their content is concatenated using the configured + separator and matched against the configured regular expression + for the replace, keep, and drop actions. + items: + type: string + type: array + targetLabel: + description: Label to which the resulting value is written + in a replace action. It is mandatory for replace actions. + Regex capture groups are available. + type: string + type: object + type: array + required: + - url + type: object + type: array + replicaExternalLabelName: + description: Name of Prometheus external label used to denote replica + name. Defaults to the value of `prometheus_replica`. External label + will _not_ be added when value is set to empty string (`""`). + type: string + replicas: + description: Number of instances to deploy for a Prometheus deployment. + format: int32 + type: integer + resources: + description: Define resources requests and limits for single Pods. + properties: + limits: + additionalProperties: + type: string + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + type: string + description: 'Requests describes the minimum amount of compute resources + required. If Requests is omitted for a container, it defaults + to Limits if that is explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + retention: + description: Time duration Prometheus shall retain data for. Default + is '24h', and must match the regular expression `[0-9]+(ms|s|m|h|d|w|y)` + (milliseconds seconds minutes hours days weeks years). + type: string + retentionSize: + description: Maximum amount of disk space used by blocks. + type: string + routePrefix: + description: The route prefix Prometheus registers HTTP handlers for. + This is useful, if using ExternalURL and a proxy is rewriting HTTP + routes of a request, and the actual ExternalURL is still true, but + the server serves requests under a different route prefix. For example + for use with `kubectl proxy`. + type: string + ruleNamespaceSelector: + description: Namespaces to be selected for PrometheusRules discovery. + If unspecified, only the same namespace as the Prometheus object is + in is used. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains + values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator + is In or NotIn, the values array must be non-empty. If the + operator is Exists or DoesNotExist, the values array must + be empty. This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator is + "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + ruleSelector: + description: A selector to select which PrometheusRules to mount for + loading alerting rules from. Until (excluding) Prometheus Operator + v0.24.0 Prometheus Operator will migrate any legacy rule ConfigMaps + to PrometheusRule custom resources selected by RuleSelector. Make + sure it does not match any config maps that you do not want to be + migrated. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains + values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator + is In or NotIn, the values array must be non-empty. If the + operator is Exists or DoesNotExist, the values array must + be empty. This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator is + "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + rules: + description: /--rules.*/ command-line arguments. + properties: + alert: + description: /--rules.alert.*/ command-line arguments + properties: + forGracePeriod: + description: Minimum duration between alert and restored 'for' + state. This is maintained only for alerts with configured + 'for' time greater than grace period. + type: string + forOutageTolerance: + description: Max time to tolerate prometheus outage for restoring + 'for' state of alert. + type: string + resendDelay: + description: Minimum amount of time to wait before resending + an alert to Alertmanager. + type: string + type: object + type: object + scrapeInterval: + description: Interval between consecutive scrapes. + type: string + secrets: + description: Secrets is a list of Secrets in the same namespace as the + Prometheus object, which shall be mounted into the Prometheus Pods. + The Secrets are mounted into /etc/prometheus/secrets/. + items: + type: string + type: array + securityContext: + description: SecurityContext holds pod-level security attributes and + common container settings. This defaults to the default PodSecurityContext. + properties: + fsGroup: + description: "A special supplemental group that applies to all containers + in a pod. Some volume types allow the Kubelet to change the ownership + of that volume to be owned by the pod: \n 1. The owning GID will + be the FSGroup 2. The setgid bit is set (new files created in + the volume will be owned by FSGroup) 3. The permission bits are + OR'd with rw-rw---- \n If unset, the Kubelet will not modify the + ownership and permissions of any volume." + format: int64 + type: integer + runAsGroup: + description: The GID to run the entrypoint of the container process. + Uses runtime default if unset. May also be set in SecurityContext. If + set in both SecurityContext and PodSecurityContext, the value + specified in SecurityContext takes precedence for that container. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a non-root + user. If true, the Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 (root) and fail to start + the container if it does. If unset or false, no such validation + will be performed. May also be set in SecurityContext. If set + in both SecurityContext and PodSecurityContext, the value specified + in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. May + also be set in SecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux + context for each container. May also be set in SecurityContext. If + set in both SecurityContext and PodSecurityContext, the value + specified in SecurityContext takes precedence for that container. + properties: + level: + description: Level is SELinux level label that applies to the + container. + type: string + role: + description: Role is a SELinux role label that applies to the + container. + type: string + type: + description: Type is a SELinux type label that applies to the + container. + type: string + user: + description: User is a SELinux user label that applies to the + container. + type: string + type: object + supplementalGroups: + description: A list of groups applied to the first process run in + each container, in addition to the container's primary GID. If + unspecified, no groups will be added to any container. + items: + format: int64 + type: integer + type: array + sysctls: + description: Sysctls hold a list of namespaced sysctls used for + the pod. Pods with unsupported sysctls (by the container runtime) + might fail to launch. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + description: The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext + will be used. If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission + webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec named by + the GMSACredentialSpecName field. This field is alpha-level + and is only honored by servers that enable the WindowsGMSA + feature flag. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA + credential spec to use. This field is alpha-level and is only + honored by servers that enable the WindowsGMSA feature flag. + type: string + runAsUserName: + description: The UserName in Windows to run the entrypoint of + the container process. Defaults to the user specified in image + metadata if unspecified. May also be set in PodSecurityContext. + If set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. This + field is beta-level and may be disabled with the WindowsRunAsUserName + feature flag. + type: string + type: object + type: object + serviceAccountName: + description: ServiceAccountName is the name of the ServiceAccount to + use to run the Prometheus Pods. + type: string + serviceMonitorNamespaceSelector: + description: Namespaces to be selected for ServiceMonitor discovery. + If nil, only check own namespace. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains + values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator + is In or NotIn, the values array must be non-empty. If the + operator is Exists or DoesNotExist, the values array must + be empty. This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator is + "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + serviceMonitorSelector: + description: ServiceMonitors to be selected for target discovery. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains + values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator + is In or NotIn, the values array must be non-empty. If the + operator is Exists or DoesNotExist, the values array must + be empty. This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator is + "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + sha: + description: SHA of Prometheus container image to be deployed. Defaults + to the value of `version`. Similar to a tag, but the SHA explicitly + deploys an immutable container image. Version and Tag are ignored + if SHA is set. + type: string + storage: + description: Storage spec to specify how storage shall be used. + properties: + emptyDir: + description: 'EmptyDirVolumeSource to be used by the Prometheus + StatefulSets. If specified, used in place of any volumeClaimTemplate. + More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir' + properties: + medium: + description: 'What type of storage medium should back this directory. + The default is "" which means to use the node''s default medium. + Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + description: 'Total amount of local storage required for this + EmptyDir volume. The size limit is also applicable for memory + medium. The maximum usage on memory medium EmptyDir would + be the minimum value between the SizeLimit specified here + and the sum of memory limits of all containers in a pod. The + default is nil which means that the limit is undefined. More + info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + type: string + type: object + volumeClaimTemplate: + description: A PVC spec to be used by the Prometheus StatefulSets. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this + representation of an object. Servers should convert recognized + schemas to the latest internal value, and may reject unrecognized + values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint + the client submits requests to. Cannot be updated. In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' + type: object + spec: + description: 'Spec defines the desired characteristics of a + volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'AccessModes contains the desired access modes + the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: This field requires the VolumeSnapshotDataSource + alpha feature gate to be enabled and currently VolumeSnapshot + is the only supported data source. If the provisioner + can support VolumeSnapshot data source, it will create + a new volume and data will be restored to the volume at + the same time. If the provisioner does not support VolumeSnapshot + data source, volume will not be created and the failure + will be reported as an event. In the future, we plan to + support more data source types and the behavior of the + provisioner may change. + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, the + specified Kind must be in the core API group. For + any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'Resources represents the minimum resources + the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + type: string + description: 'Limits describes the maximum amount of + compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + type: string + description: 'Requests describes the minimum amount + of compute resources required. If Requests is omitted + for a container, it defaults to Limits if that is + explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + selector: + description: A label query over volumes to consider for + binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + storageClassName: + description: 'Name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume is required + by the claim. Value of Filesystem is implied when not + included in claim spec. This is a beta feature. + type: string + volumeName: + description: VolumeName is the binding reference to the + PersistentVolume backing this claim. + type: string + type: object + status: + description: 'Status represents the current information/status + of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'AccessModes contains the actual access modes + the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + capacity: + additionalProperties: + type: string + description: Represents the actual resources of the underlying + volume. + type: object + conditions: + description: Current Condition of persistent volume claim. + If underlying persistent volume is being resized then + the Condition will be set to 'ResizeStarted'. + items: + description: PersistentVolumeClaimCondition contails details + about state of pvc + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transitioned + from one status to another. + format: date-time + type: string + message: + description: Human-readable message indicating details + about last transition. + type: string + reason: + description: Unique, this should be a short, machine + understandable string that gives the reason for + condition's last transition. If it reports "ResizeStarted" + that means the underlying persistent volume is being + resized. + type: string + status: + type: string + type: + description: PersistentVolumeClaimConditionType is + a valid value of PersistentVolumeClaimCondition.Type + type: string + required: + - status + - type + type: object + type: array + phase: + description: Phase represents the current phase of PersistentVolumeClaim. + type: string + type: object + type: object + type: object + tag: + description: Tag of Prometheus container image to be deployed. Defaults + to the value of `version`. Version is ignored if Tag is set. + type: string + thanos: + description: "Thanos configuration allows configuring various aspects + of a Prometheus server in a Thanos environment. \n This section is + experimental, it may change significantly without deprecation notice + in any release. \n This is experimental and may change significantly + without backward compatibility in any release." + properties: + baseImage: + description: Thanos base image if other than default. + type: string + grpcServerTlsConfig: + description: 'GRPCServerTLSConfig configures the gRPC server from + which Thanos Querier reads recorded rule data. Note: Currently + only the CAFile, CertFile, and KeyFile fields are supported. Maps + to the ''--grpc-server-tls-*'' CLI args.' + properties: + ca: + description: Stuct containing the CA cert to use for the targets. + properties: + configMap: + description: ConfigMap containing data to use for the targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + type: object + caFile: + description: Path to the CA cert in the Prometheus container + to use for the targets. + type: string + cert: + description: Struct containing the client cert file for the + targets. + properties: + configMap: + description: ConfigMap containing data to use for the targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + type: object + certFile: + description: Path to the client cert file in the Prometheus + container for the targets. + type: string + insecureSkipVerify: + description: Disable target certificate validation. + type: boolean + keyFile: + description: Path to the client key file in the Prometheus container + for the targets. + type: string + keySecret: + description: Secret containing the client key file for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + serverName: + description: Used to verify the hostname for the targets. + type: string + type: object + image: + description: Image if specified has precedence over baseImage, tag + and sha combinations. Specifying the version is still necessary + to ensure the Prometheus Operator knows what version of Thanos + is being configured. + type: string + listenLocal: + description: ListenLocal makes the Thanos sidecar listen on loopback, + so that it does not bind against the Pod IP. + type: boolean + objectStorageConfig: + description: ObjectStorageConfig configures object storage in Thanos. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + resources: + description: Resources defines the resource requirements for the + Thanos sidecar. If not provided, no requests/limits will be set + properties: + limits: + additionalProperties: + type: string + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + type: string + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + sha: + description: SHA of Thanos container image to be deployed. Defaults + to the value of `version`. Similar to a tag, but the SHA explicitly + deploys an immutable container image. Version and Tag are ignored + if SHA is set. + type: string + tag: + description: Tag of Thanos sidecar container image to be deployed. + Defaults to the value of `version`. Version is ignored if Tag + is set. + type: string + tracingConfig: + description: TracingConfig configures tracing in Thanos. This is + an experimental feature, it may change in any upcoming release + in a breaking way. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + version: + description: Version describes the version of Thanos to use. + type: string + type: object + tolerations: + description: If specified, the pod's tolerations. + items: + description: The pod this Toleration is attached to tolerates any + taint that matches the triple using the matching + operator . + properties: + effect: + description: Effect indicates the taint effect to match. Empty + means match all taint effects. When specified, allowed values + are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies + to. Empty means match all taint keys. If the key is empty, operator + must be Exists; this combination means to match all values and + all keys. + type: string + operator: + description: Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. Exists + is equivalent to wildcard for value, so that a pod can tolerate + all taints of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of time the + toleration (which must be of effect NoExecute, otherwise this + field is ignored) tolerates the taint. By default, it is not + set, which means tolerate the taint forever (do not evict). + Zero and negative values will be treated as 0 (evict immediately) + by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise + just a regular string. + type: string + type: object + type: array + version: + description: Version of Prometheus to be deployed. + type: string + volumeMounts: + description: VolumeMounts allows configuration of additional VolumeMounts + on the output StatefulSet definition. VolumeMounts specified will + be appended to other VolumeMounts in the prometheus container, that + are generated as a result of StorageSpec objects. + items: + description: VolumeMount describes a mounting of a Volume within a + container. + properties: + mountPath: + description: Path within the container at which the volume should + be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts are propagated + from the host to container and the other way around. When not + set, MountPropagationNone is used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise (false + or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the container's + volume should be mounted. Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from which the container's + volume should be mounted. Behaves similarly to SubPath but environment + variable references $(VAR_NAME) are expanded using the container's + environment. Defaults to "" (volume's root). SubPathExpr and + SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + volumes: + description: Volumes allows configuration of additional volumes on the + output StatefulSet definition. Volumes specified will be appended + to other volumes that are generated as a result of StorageSpec objects. + items: + description: Volume represents a named volume in a pod that may be + accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: 'AWSElasticBlockStore represents an AWS Disk resource + that is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + properties: + fsType: + description: 'Filesystem type of the volume that you want + to mount. Tip: Ensure that the filesystem type is supported + by the host operating system. Examples: "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem from compromising + the machine' + type: string + partition: + description: 'The partition in the volume that you want to + mount. If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition + as "1". Similarly, the volume partition for /dev/sda is + "0" (or you can leave the property empty).' + format: int32 + type: integer + readOnly: + description: 'Specify "true" to force and set the ReadOnly + property in VolumeMounts to "true". If omitted, the default + is "false". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'Unique ID of the persistent disk resource in + AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID + type: object + azureDisk: + description: AzureDisk represents an Azure Data Disk mount on + the host and bind mount to the pod. + properties: + cachingMode: + description: 'Host Caching mode: None, Read Only, Read Write.' + type: string + diskName: + description: The Name of the data disk in the blob storage + type: string + diskURI: + description: The URI the data disk in the blob storage + type: string + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'Expected values Shared: multiple blob disks + per storage account Dedicated: single blob disk per storage + account Managed: azure managed data disk (only in managed + availability set). defaults to shared' + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly here + will force the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: AzureFile represents an Azure File Service mount + on the host and bind mount to the pod. + properties: + readOnly: + description: Defaults to false (read/write). ReadOnly here + will force the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: the name of secret that contains Azure Storage + Account Name and Key + type: string + shareName: + description: Share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: CephFS represents a Ceph FS mount on the host that + shares a pod's lifetime + properties: + monitors: + description: 'Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'Optional: Used as the mounted root, rather than + the full Ceph tree, default is /' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. More + info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'Optional: SecretFile is the path to key ring + for User, default is /etc/ceph/user.secret More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'Optional: SecretRef is reference to the authentication + secret for User, default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + user: + description: 'Optional: User is the rados user name, default + is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors + type: object + cinder: + description: 'Cinder represents a cinder volume attached and mounted + on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Examples: "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. More + info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'Optional: points to a secret object containing + parameters used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + volumeID: + description: 'volume id used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + required: + - volumeID + type: object + configMap: + description: ConfigMap represents a configMap that should populate + this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created files + by default. Must be a value between 0 and 0777. Defaults + to 0644. Directories within the path are not affected by + this setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair in the Data + field of the referenced ConfigMap will be projected into + the volume as a file whose name is the key and content is + the value. If specified, the listed keys will be projected + into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the + ConfigMap, the volume setup will error unless it is marked + optional. Paths must be relative and may not contain the + '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to use on this file, + must be a value between 0 and 0777. If not specified, + the volume defaultMode will be used. This might be + in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode + bits set.' + format: int32 + type: integer + path: + description: The relative path of the file to map the + key to. May not be an absolute path. May not contain + the path element '..'. May not start with the string + '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its keys must + be defined + type: boolean + type: object + csi: + description: CSI (Container Storage Interface) represents storage + that is handled by an external CSI driver (Alpha feature). + properties: + driver: + description: Driver is the name of the CSI driver that handles + this volume. Consult with your admin for the correct name + as registered in the cluster. + type: string + fsType: + description: Filesystem type to mount. Ex. "ext4", "xfs", + "ntfs". If not provided, the empty value is passed to the + associated CSI driver which will determine the default filesystem + to apply. + type: string + nodePublishSecretRef: + description: NodePublishSecretRef is a reference to the secret + object containing sensitive information to pass to the CSI + driver to complete the CSI NodePublishVolume and NodeUnpublishVolume + calls. This field is optional, and may be empty if no secret + is required. If the secret object contains more than one + secret, all secret references are passed. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + readOnly: + description: Specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: VolumeAttributes stores driver-specific properties + that are passed to the CSI driver. Consult your driver's + documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: DownwardAPI represents downward API about the pod + that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created files + by default. Must be a value between 0 and 0777. Defaults + to 0644. Directories within the path are not affected by + this setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: + only annotations, labels, name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits to use on this file, + must be a value between 0 and 0777. If not specified, + the volume defaultMode will be used. This might be + in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode + bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative path name + of the file to be created. Must not be absolute or + contain the ''..'' path. Must be utf-8 encoded. The + first item of the relative path must not start with + ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + requests.cpu and requests.memory) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + description: Specifies the output format of the + exposed resources, defaults to "1" + type: string + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + description: 'EmptyDir represents a temporary directory that shares + a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'What type of storage medium should back this + directory. The default is "" which means to use the node''s + default medium. Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + description: 'Total amount of local storage required for this + EmptyDir volume. The size limit is also applicable for memory + medium. The maximum usage on memory medium EmptyDir would + be the minimum value between the SizeLimit specified here + and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + type: string + type: object + fc: + description: FC represents a Fibre Channel resource that is attached + to a kubelet's host machine and then exposed to the pod. + properties: + fsType: + description: 'Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + TODO: how do we prevent errors in the filesystem from compromising + the machine' + type: string + lun: + description: 'Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'Optional: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts.' + type: boolean + targetWWNs: + description: 'Optional: FC target worldwide names (WWNs)' + items: + type: string + type: array + wwids: + description: 'Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be + set, but not both simultaneously.' + items: + type: string + type: array + type: object + flexVolume: + description: FlexVolume represents a generic volume resource that + is provisioned/attached using an exec based plugin. + properties: + driver: + description: Driver is the name of the driver to use for this + volume. + type: string + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". The default filesystem depends on FlexVolume + script. + type: string + options: + additionalProperties: + type: string + description: 'Optional: Extra command options if any.' + type: object + readOnly: + description: 'Optional: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts.' + type: boolean + secretRef: + description: 'Optional: SecretRef is reference to the secret + object containing sensitive information to pass to the plugin + scripts. This may be empty if no secret object is specified. + If the secret object contains more than one secret, all + secrets are passed to the plugin scripts.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + required: + - driver + type: object + flocker: + description: Flocker represents a Flocker volume attached to a + kubelet's host machine. This depends on the Flocker control + service being running + properties: + datasetName: + description: Name of the dataset stored as metadata -> name + on the dataset for Flocker should be considered as deprecated + type: string + datasetUUID: + description: UUID of the dataset. This is unique identifier + of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'GCEPersistentDisk represents a GCE Disk resource + that is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'Filesystem type of the volume that you want + to mount. Tip: Ensure that the filesystem type is supported + by the host operating system. Examples: "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem from compromising + the machine' + type: string + partition: + description: 'The partition in the volume that you want to + mount. If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition + as "1". Similarly, the volume partition for /dev/sda is + "0" (or you can leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'Unique name of the PD resource in GCE. Used + to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'GitRepo represents a git repository at a particular + revision. DEPRECATED: GitRepo is deprecated. To provision a + container with a git repo, mount an EmptyDir into an InitContainer + that clones the repo using git, then mount the EmptyDir into + the Pod''s container.' + properties: + directory: + description: Target directory name. Must not contain or start + with '..'. If '.' is supplied, the volume directory will + be the git repository. Otherwise, if specified, the volume + will contain the git repository in the subdirectory with + the given name. + type: string + repository: + description: Repository URL + type: string + revision: + description: Commit hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'Glusterfs represents a Glusterfs mount on the host + that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'EndpointsName is the endpoint name that details + Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'Path is the Glusterfs volume path. More info: + https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'ReadOnly here will force the Glusterfs volume + to be mounted with read-only permissions. Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'HostPath represents a pre-existing file or directory + on the host machine that is directly exposed to the container. + This is generally used for system agents or other privileged + things that are allowed to see the host machine. Most containers + will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use host directory + mounts and who can/can not mount host directories as read/write.' + properties: + path: + description: 'Path of the directory on the host. If the path + is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'Type for HostPath Volume Defaults to "" More + info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'ISCSI represents an ISCSI Disk resource that is + attached to a kubelet''s host machine and then exposed to the + pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: whether support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: whether support iSCSI Session CHAP authentication + type: boolean + fsType: + description: 'Filesystem type of the volume that you want + to mount. Tip: Ensure that the filesystem type is supported + by the host operating system. Examples: "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem from compromising + the machine' + type: string + initiatorName: + description: Custom iSCSI Initiator Name. If initiatorName + is specified with iscsiInterface simultaneously, new iSCSI + interface : will be created + for the connection. + type: string + iqn: + description: Target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iSCSI Interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: iSCSI Target Portal List. The portal is either + an IP or ip_addr:port if the port is other than default + (typically TCP ports 860 and 3260). + items: + type: string + type: array + readOnly: + description: ReadOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: CHAP Secret for iSCSI target and initiator authentication + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + targetPortal: + description: iSCSI Target Portal. The Portal is either an + IP or ip_addr:port if the port is other than default (typically + TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'Volume''s name. Must be a DNS_LABEL and unique within + the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + nfs: + description: 'NFS represents an NFS mount on the host that shares + a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'Path that is exported by the NFS server. More + info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'ReadOnly here will force the NFS export to be + mounted with read-only permissions. Defaults to false. More + info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'Server is the hostname or IP address of the + NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'PersistentVolumeClaimVolumeSource represents a reference + to a PersistentVolumeClaim in the same namespace. More info: + https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + claimName: + description: 'ClaimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: PhotonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host machine + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: ID that identifies Photon Controller persistent + disk + type: string + required: + - pdID + type: object + portworxVolume: + description: PortworxVolume represents a portworx volume attached + and mounted on kubelets host machine + properties: + fsType: + description: FSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs". Implicitly inferred to be "ext4" + if unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly here + will force the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: VolumeID uniquely identifies a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: Items for all in one resources secrets, configmaps, + and downward API + properties: + defaultMode: + description: Mode bits to use on created files by default. + Must be a value between 0 and 0777. Directories within the + path are not affected by this setting. This might be in + conflict with other options that affect the file mode, like + fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: list of volume projections + items: + description: Projection that may be projected along with + other supported volume types + properties: + configMap: + description: information about the configMap data to + project + properties: + items: + description: If unspecified, each key-value pair + in the Data field of the referenced ConfigMap + will be projected into the volume as a file whose + name is the key and content is the value. If specified, + the listed keys will be projected into the specified + paths, and unlisted keys will not be present. + If a key is specified which is not present in + the ConfigMap, the volume setup will error unless + it is marked optional. Paths must be relative + and may not contain the '..' path or start with + '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to use on + this file, must be a value between 0 and + 0777. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the file + to map the key to. May not be an absolute + path. May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + keys must be defined + type: boolean + type: object + downwardAPI: + description: information about the downwardAPI data + to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing the + pod field + properties: + fieldRef: + description: 'Required: Selects a field of + the pod: only annotations, labels, name + and namespace are supported.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits to use on + this file, must be a value between 0 and + 0777. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must + not be absolute or contain the ''..'' path. + Must be utf-8 encoded. The first item of + the relative path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + description: Specifies the output format + of the exposed resources, defaults to + "1" + type: string + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + description: information about the secret data to project + properties: + items: + description: If unspecified, each key-value pair + in the Data field of the referenced Secret will + be projected into the volume as a file whose name + is the key and content is the value. If specified, + the listed keys will be projected into the specified + paths, and unlisted keys will not be present. + If a key is specified which is not present in + the Secret, the volume setup will error unless + it is marked optional. Paths must be relative + and may not contain the '..' path or start with + '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to use on + this file, must be a value between 0 and + 0777. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the file + to map the key to. May not be an absolute + path. May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + type: object + serviceAccountToken: + description: information about the serviceAccountToken + data to project + properties: + audience: + description: Audience is the intended audience of + the token. A recipient of a token must identify + itself with an identifier specified in the audience + of the token, and otherwise should reject the + token. The audience defaults to the identifier + of the apiserver. + type: string + expirationSeconds: + description: ExpirationSeconds is the requested + duration of validity of the service account token. + As the token approaches expiration, the kubelet + volume plugin will proactively rotate the service + account token. The kubelet will start trying to + rotate the token if the token is older than 80 + percent of its time to live or if the token is + older than 24 hours.Defaults to 1 hour and must + be at least 10 minutes. + format: int64 + type: integer + path: + description: Path is the path relative to the mount + point of the file to project the token into. + type: string + required: + - path + type: object + type: object + type: array + required: + - sources + type: object + quobyte: + description: Quobyte represents a Quobyte mount on the host that + shares a pod's lifetime + properties: + group: + description: Group to map volume access to Default is no group + type: string + readOnly: + description: ReadOnly here will force the Quobyte volume to + be mounted with read-only permissions. Defaults to false. + type: boolean + registry: + description: Registry represents a single or multiple Quobyte + Registry services specified as a string as host:port pair + (multiple entries are separated with commas) which acts + as the central registry for volumes + type: string + tenant: + description: Tenant owning the given Quobyte volume in the + Backend Used with dynamically provisioned Quobyte volumes, + value is set by the plugin + type: string + user: + description: User to map volume access to Defaults to serivceaccount + user + type: string + volume: + description: Volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'RBD represents a Rados Block Device mount on the + host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' + properties: + fsType: + description: 'Filesystem type of the volume that you want + to mount. Tip: Ensure that the filesystem type is supported + by the host operating system. Examples: "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem from compromising + the machine' + type: string + image: + description: 'The rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'Keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'A collection of Ceph monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'The rados pool name. Default is rbd. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'SecretRef is name of the authentication secret + for RBDUser. If provided overrides keyring. Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + user: + description: 'The rados user name. Default is admin. More + info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: ScaleIO represents a ScaleIO persistent volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Default is "xfs". + type: string + gateway: + description: The host address of the ScaleIO API Gateway. + type: string + protectionDomain: + description: The name of the ScaleIO Protection Domain for + the configured storage. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly here + will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef references to the secret for ScaleIO + user and other sensitive information. If this is not provided, + Login operation will fail. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + sslEnabled: + description: Flag to enable/disable SSL communication with + Gateway, default false + type: boolean + storageMode: + description: Indicates whether the storage for a volume should + be ThickProvisioned or ThinProvisioned. Default is ThinProvisioned. + type: string + storagePool: + description: The ScaleIO Storage Pool associated with the + protection domain. + type: string + system: + description: The name of the storage system as configured + in ScaleIO. + type: string + volumeName: + description: The name of a volume already created in the ScaleIO + system that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'Secret represents a secret that should populate + this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'Optional: mode bits to use on created files + by default. Must be a value between 0 and 0777. Defaults + to 0644. Directories within the path are not affected by + this setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair in the Data + field of the referenced Secret will be projected into the + volume as a file whose name is the key and content is the + value. If specified, the listed keys will be projected into + the specified paths, and unlisted keys will not be present. + If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. + Paths must be relative and may not contain the '..' path + or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to use on this file, + must be a value between 0 and 0777. If not specified, + the volume defaultMode will be used. This might be + in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode + bits set.' + format: int32 + type: integer + path: + description: The relative path of the file to map the + key to. May not be an absolute path. May not contain + the path element '..'. May not start with the string + '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: Specify whether the Secret or its keys must be + defined + type: boolean + secretName: + description: 'Name of the secret in the pod''s namespace to + use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: StorageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly here + will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef specifies the secret to use for obtaining + the StorageOS API credentials. If not specified, default + values will be attempted. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + volumeName: + description: VolumeName is the human-readable name of the + StorageOS volume. Volume names are only unique within a + namespace. + type: string + volumeNamespace: + description: VolumeNamespace specifies the scope of the volume + within StorageOS. If no namespace is specified then the + Pod's namespace will be used. This allows the Kubernetes + name scoping to be mirrored within StorageOS for tighter + integration. Set VolumeName to any name to override the + default behaviour. Set to "default" if you are not using + namespaces within StorageOS. Namespaces that do not pre-exist + within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: VsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: Storage Policy Based Management (SPBM) profile + ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: Storage Policy Based Management (SPBM) profile + name. + type: string + volumePath: + description: Path that identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + walCompression: + description: Enable compression of the write-ahead log using Snappy. + This flag is only available in versions of Prometheus >= 2.11.0. + type: boolean + type: object + status: + description: 'Most recent observed status of the Prometheus cluster. Read-only. + Not included when requesting from the apiserver, only from the Prometheus + Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' + properties: + availableReplicas: + description: Total number of available pods (ready for at least minReadySeconds) + targeted by this Prometheus deployment. + format: int32 + type: integer + paused: + description: Represents whether any actions on the underlaying managed + objects are being performed. Only delete actions will be performed. + type: boolean + replicas: + description: Total number of non-terminated pods targeted by this Prometheus + deployment (their labels match the selector). + format: int32 + type: integer + unavailableReplicas: + description: Total number of unavailable pods targeted by this Prometheus + deployment. + format: int32 + type: integer + updatedReplicas: + description: Total number of non-terminated pods targeted by this Prometheus + deployment that have the desired version spec. + format: int32 + type: integer + required: + - availableReplicas + - paused + - replicas + - unavailableReplicas + - updatedReplicas + type: object + required: + - spec + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true diff --git a/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-prometheusrules.yaml b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-prometheusrules.yaml new file mode 100755 index 000000000..5546de38e --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-prometheusrules.yaml @@ -0,0 +1,91 @@ +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.38/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.4 + creationTimestamp: null + name: prometheusrules.monitoring.coreos.com +spec: + group: monitoring.coreos.com + names: + kind: PrometheusRule + listKind: PrometheusRuleList + plural: prometheusrules + singular: prometheusrule + preserveUnknownFields: false + scope: Namespaced + validation: + openAPIV3Schema: + description: PrometheusRule defines alerting rules for a Prometheus instance + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Specification of desired alerting rule definitions for Prometheus. + properties: + groups: + description: Content of Prometheus rule file + items: + description: 'RuleGroup is a list of sequentially evaluated recording + and alerting rules. Note: PartialResponseStrategy is only used by + ThanosRuler and will be ignored by Prometheus instances. Valid + values for this field are ''warn'' or ''abort''. More info: https://github.com/thanos-io/thanos/blob/master/docs/components/rule.md#partial-response' + properties: + interval: + type: string + name: + type: string + partial_response_strategy: + type: string + rules: + items: + description: Rule describes an alerting or recording rule. + properties: + alert: + type: string + annotations: + additionalProperties: + type: string + type: object + expr: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + for: + type: string + labels: + additionalProperties: + type: string + type: object + record: + type: string + required: + - expr + type: object + type: array + required: + - name + - rules + type: object + type: array + type: object + required: + - spec + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true diff --git a/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-servicemonitor.yaml new file mode 100755 index 000000000..8f7a67c14 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-servicemonitor.yaml @@ -0,0 +1,459 @@ +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.38/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.4 + creationTimestamp: null + name: servicemonitors.monitoring.coreos.com +spec: + group: monitoring.coreos.com + names: + kind: ServiceMonitor + listKind: ServiceMonitorList + plural: servicemonitors + singular: servicemonitor + preserveUnknownFields: false + scope: Namespaced + validation: + openAPIV3Schema: + description: ServiceMonitor defines monitoring for a set of services. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Specification of desired Service selection for target discovery + by Prometheus. + properties: + endpoints: + description: A list of endpoints allowed as part of this ServiceMonitor. + items: + description: Endpoint defines a scrapeable endpoint serving Prometheus + metrics. + properties: + basicAuth: + description: 'BasicAuth allow an endpoint to authenticate over + basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints' + properties: + password: + description: The secret in the service monitor namespace that + contains the password for authentication. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + username: + description: The secret in the service monitor namespace that + contains the username for authentication. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + type: object + bearerTokenFile: + description: File to read bearer token for scraping targets. + type: string + bearerTokenSecret: + description: Secret to mount to read bearer token for scraping + targets. The secret needs to be in the same namespace as the + service monitor and accessible by the Prometheus Operator. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + honorLabels: + description: HonorLabels chooses the metric's labels on collisions + with target labels. + type: boolean + honorTimestamps: + description: HonorTimestamps controls whether Prometheus respects + the timestamps present in scraped data. + type: boolean + interval: + description: Interval at which metrics should be scraped + type: string + metricRelabelings: + description: MetricRelabelConfigs to apply to samples before ingestion. + items: + description: 'RelabelConfig allows dynamic rewriting of the + label set, being applied to samples before ingestion. It defines + ``-section of Prometheus configuration. + More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' + properties: + action: + description: Action to perform based on regex matching. + Default is 'replace' + type: string + modulus: + description: Modulus to take of the hash of the source label + values. + format: int64 + type: integer + regex: + description: Regular expression against which the extracted + value is matched. Default is '(.*)' + type: string + replacement: + description: Replacement value against which a regex replace + is performed if the regular expression matches. Regex + capture groups are available. Default is '$1' + type: string + separator: + description: Separator placed between concatenated source + label values. default is ';'. + type: string + sourceLabels: + description: The source labels select values from existing + labels. Their content is concatenated using the configured + separator and matched against the configured regular expression + for the replace, keep, and drop actions. + items: + type: string + type: array + targetLabel: + description: Label to which the resulting value is written + in a replace action. It is mandatory for replace actions. + Regex capture groups are available. + type: string + type: object + type: array + params: + additionalProperties: + items: + type: string + type: array + description: Optional HTTP URL parameters + type: object + path: + description: HTTP path to scrape for metrics. + type: string + port: + description: Name of the service port this endpoint refers to. + Mutually exclusive with targetPort. + type: string + proxyUrl: + description: ProxyURL eg http://proxyserver:2195 Directs scrapes + to proxy through this endpoint. + type: string + relabelings: + description: 'RelabelConfigs to apply to samples before scraping. + More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config' + items: + description: 'RelabelConfig allows dynamic rewriting of the + label set, being applied to samples before ingestion. It defines + ``-section of Prometheus configuration. + More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs' + properties: + action: + description: Action to perform based on regex matching. + Default is 'replace' + type: string + modulus: + description: Modulus to take of the hash of the source label + values. + format: int64 + type: integer + regex: + description: Regular expression against which the extracted + value is matched. Default is '(.*)' + type: string + replacement: + description: Replacement value against which a regex replace + is performed if the regular expression matches. Regex + capture groups are available. Default is '$1' + type: string + separator: + description: Separator placed between concatenated source + label values. default is ';'. + type: string + sourceLabels: + description: The source labels select values from existing + labels. Their content is concatenated using the configured + separator and matched against the configured regular expression + for the replace, keep, and drop actions. + items: + type: string + type: array + targetLabel: + description: Label to which the resulting value is written + in a replace action. It is mandatory for replace actions. + Regex capture groups are available. + type: string + type: object + type: array + scheme: + description: HTTP scheme to use for scraping. + type: string + scrapeTimeout: + description: Timeout after which the scrape is ended + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: Name or number of the pod port this endpoint refers + to. Mutually exclusive with port. + x-kubernetes-int-or-string: true + tlsConfig: + description: TLS configuration to use when scraping the endpoint + properties: + ca: + description: Stuct containing the CA cert to use for the targets. + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + caFile: + description: Path to the CA cert in the Prometheus container + to use for the targets. + type: string + cert: + description: Struct containing the client cert file for the + targets. + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + certFile: + description: Path to the client cert file in the Prometheus + container for the targets. + type: string + insecureSkipVerify: + description: Disable target certificate validation. + type: boolean + keyFile: + description: Path to the client key file in the Prometheus + container for the targets. + type: string + keySecret: + description: Secret containing the client key file for the + targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + serverName: + description: Used to verify the hostname for the targets. + type: string + type: object + type: object + type: array + jobLabel: + description: The label to use to retrieve the job name from. + type: string + namespaceSelector: + description: Selector to select which namespaces the Endpoints objects + are discovered from. + properties: + any: + description: Boolean describing whether all namespaces are selected + in contrast to a list restricting them. + type: boolean + matchNames: + description: List of namespace names. + items: + type: string + type: array + type: object + podTargetLabels: + description: PodTargetLabels transfers labels on the Kubernetes Pod + onto the target. + items: + type: string + type: array + sampleLimit: + description: SampleLimit defines per-scrape limit on number of scraped + samples that will be accepted. + format: int64 + type: integer + selector: + description: Selector to select Endpoints objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains + values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator + is In or NotIn, the values array must be non-empty. If the + operator is Exists or DoesNotExist, the values array must + be empty. This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator is + "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + targetLabels: + description: TargetLabels transfers labels on the Kubernetes Service + onto the target. + items: + type: string + type: array + required: + - endpoints + - selector + type: object + required: + - spec + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true diff --git a/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-thanosrulers.yaml b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-thanosrulers.yaml new file mode 100755 index 000000000..82136d73e --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/crd-manifest/crd-thanosrulers.yaml @@ -0,0 +1,4725 @@ +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.38/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.4 + creationTimestamp: null + name: thanosrulers.monitoring.coreos.com +spec: + group: monitoring.coreos.com + names: + kind: ThanosRuler + listKind: ThanosRulerList + plural: thanosrulers + singular: thanosruler + preserveUnknownFields: false + scope: Namespaced + validation: + openAPIV3Schema: + description: ThanosRuler defines a ThanosRuler deployment. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: 'Specification of the desired behavior of the ThanosRuler cluster. + More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' + properties: + affinity: + description: If specified, the pod's scheduling constraints. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes + that satisfy the affinity expressions specified by this field, + but it may choose a node that violates one or more of the + expressions. The node that is most preferred is the one with + the greatest sum of weights, i.e. for each node that meets + all of the scheduling requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating through + the elements of this field and adding "weight" to the sum + if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: An empty preferred scheduling term matches all + objects with implicit weight 0 (i.e. it's a no-op). A null + preferred scheduling term matches no objects (i.e. is also + a no-op). + properties: + preference: + description: A node selector term, associated with the + corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the + operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be + empty. If the operator is Gt or Lt, the values + array must have a single element, which will + be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the + operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be + empty. If the operator is Gt or Lt, the values + array must have a single element, which will + be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by this + field are not met at scheduling time, the pod will not be + scheduled onto the node. If the affinity requirements specified + by this field cease to be met at some point during pod execution + (e.g. due to an update), the system may or may not try to + eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The + terms are ORed. + items: + description: A null or empty node selector term matches + no objects. The requirements of them are ANDed. The + TopologySelectorTerm type implements a subset of the + NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the + operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be + empty. If the operator is Gt or Lt, the values + array must have a single element, which will + be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the + operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be + empty. If the operator is Gt or Lt, the values + array must have a single element, which will + be interpreted as an integer. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes + that satisfy the affinity expressions specified by this field, + but it may choose a node that violates one or more of the + expressions. The node that is most preferred is the one with + the greatest sum of weights, i.e. for each node that meets + all of the scheduling requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating through + the elements of this field and adding "weight" to the sum + if the node has pods which matches the corresponding podAffinityTerm; + the node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement is + a selector that contains values, a key, and + an operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. This array + is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified namespaces, + where co-located is defined as running on a node + whose value of the label with key topologyKey matches + that of any node on which any of the selected pods + is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the corresponding + podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by this + field are not met at scheduling time, the pod will not be + scheduled onto the node. If the affinity requirements specified + by this field cease to be met at some point during pod execution + (e.g. due to a pod label update), the system may or may not + try to eventually evict the pod from its node. When there + are multiple elements, the lists of nodes corresponding to + each podAffinityTerm are intersected, i.e. all terms must + be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) that + this pod should be co-located (affinity) or not co-located + (anti-affinity) with, where co-located is defined as running + on a node whose value of the label with key + matches that of any node on which a pod of the set of pods + is running + properties: + labelSelector: + description: A label query over a set of resources, in + this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values + array must be non-empty. If the operator is + Exists or DoesNotExist, the values array must + be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces the + labelSelector applies to (matches against); null or + empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where + co-located is defined as running on a node whose value + of the label with key topologyKey matches that of any + node on which any of the selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some other + pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes + that satisfy the anti-affinity expressions specified by this + field, but it may choose a node that violates one or more + of the expressions. The node that is most preferred is the + one with the greatest sum of weights, i.e. for each node that + meets all of the scheduling requirements (resource request, + requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field + and adding "weight" to the sum if the node has pods which + matches the corresponding podAffinityTerm; the node(s) with + the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement is + a selector that contains values, a key, and + an operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. This array + is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified namespaces, + where co-located is defined as running on a node + whose value of the label with key topologyKey matches + that of any node on which any of the selected pods + is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the corresponding + podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements specified by + this field are not met at scheduling time, the pod will not + be scheduled onto the node. If the anti-affinity requirements + specified by this field cease to be met at some point during + pod execution (e.g. due to a pod label update), the system + may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding + to each podAffinityTerm are intersected, i.e. all terms must + be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) that + this pod should be co-located (affinity) or not co-located + (anti-affinity) with, where co-located is defined as running + on a node whose value of the label with key + matches that of any node on which a pod of the set of pods + is running + properties: + labelSelector: + description: A label query over a set of resources, in + this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values + array must be non-empty. If the operator is + Exists or DoesNotExist, the values array must + be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces the + labelSelector applies to (matches against); null or + empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where + co-located is defined as running on a node whose value + of the label with key topologyKey matches that of any + node on which any of the selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + alertDropLabels: + description: AlertDropLabels configure the label names which should + be dropped in ThanosRuler alerts. If `labels` field is not provided, + `thanos_ruler_replica` will be dropped in alerts by default. + items: + type: string + type: array + alertQueryUrl: + description: The external Query URL the Thanos Ruler will set in the + 'Source' field of all alerts. Maps to the '--alert.query-url' CLI + arg. + type: string + alertmanagersConfig: + description: Define configuration for connecting to alertmanager. Only + available with thanos v0.10.0 and higher. Maps to the `alertmanagers.config` + arg. + properties: + key: + description: The key of the secret to select from. Must be a valid + secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + alertmanagersUrl: + description: 'Define URLs to send alerts to Alertmanager. For Thanos + v0.10.0 and higher, AlertManagersConfig should be used instead. Note: + this field will be ignored if AlertManagersConfig is specified. Maps + to the `alertmanagers.url` arg.' + items: + type: string + type: array + containers: + description: 'Containers allows injecting additional containers or modifying + operator generated containers. This can be used to allow adding an + authentication proxy to a ThanosRuler pod or to change the behavior + of an operator generated container. Containers described here modify + an operator generated container if they share the same name and modifications + are done via a strategic merge patch. The current container names + are: `thanos-ruler` and `rules-configmap-reloader`. Overriding containers + is entirely outside the scope of what the maintainers will support + and by doing so, you accept that this behaviour may break at any time + without notice.' + items: + description: A single application container that you want to run within + a pod. + properties: + args: + description: 'Arguments to the entrypoint. The docker image''s + CMD is used if this is not provided. Variable references $(VAR_NAME) + are expanded using the container''s environment. If a variable + cannot be resolved, the reference in the input string will be + unchanged. The $(VAR_NAME) syntax can be escaped with a double + $$, ie: $$(VAR_NAME). Escaped references will never be expanded, + regardless of whether the variable exists or not. Cannot be + updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within a shell. The + docker image''s ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, the reference + in the input string will be unchanged. The $(VAR_NAME) syntax + can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable exists + or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previous defined environment variables in the + container and any service environment variables. If a + variable cannot be resolved, the reference in the input + string will be unchanged. The $(VAR_NAME) syntax can be + escaped with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable + exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, metadata.labels, metadata.annotations, + spec.nodeName, spec.serviceAccountName, status.hostIP, + status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + description: Specifies the output format of the + exposed resources, defaults to "1" + type: string + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment variables + in the container. The keys defined within a source must be a + C_IDENTIFIER. All invalid keys will be reported as an event + when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take + precedence. Values defined by an Env with a duplicate key will + take precedence. Cannot be updated. + items: + description: EnvFromSource represents the source of a set of + ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + prefix: + description: An optional identifier to prepend to each key + in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + type: object + type: array + image: + description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management + to default or override container images in workload controllers + like Deployments and StatefulSets.' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent + otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Actions that the management system should take in + response to container lifecycle events. Cannot be updated. + properties: + postStart: + description: 'PostStart is called immediately after a container + is created. If the handler fails, the container is terminated + and restarted according to its restart policy. Other management + of the container blocks until the hook completes. More info: + https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's filesystem. + The command is simply exec'd, it is not run inside + a shell, so traditional shell instructions ('|', + etc) won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is treated + as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately before a container + is terminated due to an API request or management event + such as liveness/startup probe failure, preemption, resource + contention, etc. The handler is not called if the container + crashes or exits. The reason for termination is passed to + the handler. The Pod''s termination grace period countdown + begins before the PreStop hooked is executed. Regardless + of the outcome of the handler, the container will eventually + terminate within the Pod''s termination grace period. Other + management of the container blocks until the hook completes + or until the termination grace period is reached. More info: + https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's filesystem. + The command is simply exec'd, it is not run inside + a shell, so traditional shell instructions ('|', + etc) won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is treated + as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: 'Periodic probe of container liveness. Container + will be restarted if the probe fails. Cannot be updated. More + info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + name: + description: Name of the container specified as a DNS_LABEL. Each + container in a pod must have a unique name (DNS_LABEL). Cannot + be updated. + type: string + ports: + description: List of ports to expose from the container. Exposing + a port here gives the system additional information about the + network connections a container uses, but is primarily informational. + Not specifying a port here DOES NOT prevent that port from being + exposed. Any port which is listening on the default "0.0.0.0" + address inside a container will be accessible from the network. + Cannot be updated. + items: + description: ContainerPort represents a network port in a single + container. + properties: + containerPort: + description: Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: Number of port to expose on the host. If specified, + this must be a valid port number, 0 < x < 65536. If HostNetwork + is specified, this must match ContainerPort. Most containers + do not need this. + format: int32 + type: integer + name: + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port in a pod must + have a unique name. Name for the port that can be referred + to by services. + type: string + protocol: + description: Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + readinessProbe: + description: 'Periodic probe of container service readiness. Container + will be removed from service endpoints if the probe fails. Cannot + be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + resources: + description: 'Compute Resources required by this container. Cannot + be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + properties: + limits: + additionalProperties: + type: string + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + type: string + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + securityContext: + description: 'Security options the pod should run with. More info: + https://kubernetes.io/docs/concepts/policy/security-context/ + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether a + process can gain more privileges than its parent process. + This bool directly controls if the no_new_privs flag will + be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as Privileged + 2) has CAP_SYS_ADMIN' + type: boolean + capabilities: + description: The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the + container runtime. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes in + privileged containers are essentially equivalent to root + on the host. Defaults to false. + type: boolean + procMount: + description: procMount denotes the type of proc mount to use + for the containers. The default is DefaultProcMount which + uses the container runtime defaults for readonly paths and + masked paths. This requires the ProcMountType feature flag + to be enabled. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only root filesystem. + Default is false. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container + process. Uses runtime default if unset. May also be set + in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a non-root + user. If true, the Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 (root) and fail + to start the container if it does. If unset or false, no + such validation will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container + process. Defaults to user specified in image metadata if + unspecified. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random + SELinux context for each container. May also be set in + PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + windowsOptions: + description: The Windows specific settings applied to all + containers. If unspecified, the options from the PodSecurityContext + will be used. If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission + webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec named + by the GMSACredentialSpecName field. This field is alpha-level + and is only honored by servers that enable the WindowsGMSA + feature flag. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. This field is alpha-level + and is only honored by servers that enable the WindowsGMSA + feature flag. + type: string + runAsUserName: + description: The UserName in Windows to run the entrypoint + of the container process. Defaults to the user specified + in image metadata if unspecified. May also be set in + PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. This field is beta-level and may be + disabled with the WindowsRunAsUserName feature flag. + type: string + type: object + type: object + startupProbe: + description: 'StartupProbe indicates that the Pod has successfully + initialized. If specified, no other probes are executed until + this completes successfully. If this probe fails, the Pod will + be restarted, just as if the livenessProbe failed. This can + be used to provide different probe parameters at the beginning + of a Pod''s lifecycle, when it might take a long time to load + data or warm a cache, than during steady-state operation. This + cannot be updated. This is an alpha feature enabled by the StartupProbe + feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + stdin: + description: Whether this container should allocate a buffer for + stdin in the container runtime. If this is not set, reads from + stdin in the container will always result in EOF. Default is + false. + type: boolean + stdinOnce: + description: Whether the container runtime should close the stdin + channel after it has been opened by a single attach. When stdin + is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container + start, is empty until the first client attaches to stdin, and + then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container + is restarted. If this flag is false, a container processes that + reads from stdin will never receive an EOF. Default is false + type: boolean + terminationMessagePath: + description: 'Optional: Path at which the file to which the container''s + termination message will be written is mounted into the container''s + filesystem. Message written is intended to be brief final status, + such as an assertion failure message. Will be truncated by the + node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. Defaults to /dev/termination-log. + Cannot be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message should be populated. + File will use the contents of terminationMessagePath to populate + the container status message on both success and failure. FallbackToLogsOnError + will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever + is smaller. Defaults to File. Cannot be updated. + type: string + tty: + description: Whether this container should allocate a TTY for + itself, also requires 'stdin' to be true. Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be + used by the container. This is a beta feature. + items: + description: volumeDevice describes a mapping of a raw block + device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container + that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: Path within the container at which the volume + should be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts are + propagated from the host to container and the other way + around. When not set, MountPropagationNone is used. This + field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the container's + volume should be mounted. Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from which + the container's volume should be mounted. Behaves similarly + to SubPath but environment variable references $(VAR_NAME) + are expanded using the container's environment. Defaults + to "" (volume's root). SubPathExpr and SubPath are mutually + exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If not specified, + the container runtime's default will be used, which might be + configured in the container image. Cannot be updated. + type: string + required: + - name + type: object + type: array + enforcedNamespaceLabel: + description: EnforcedNamespaceLabel enforces adding a namespace label + of origin for each alert and metric that is user created. The label + value will always be the namespace of the object that is being created. + type: string + evaluationInterval: + description: Interval between consecutive evaluations. + type: string + externalPrefix: + description: The external URL the Thanos Ruler instances will be available + under. This is necessary to generate correct URLs. This is necessary + if Thanos Ruler is not served from root of a DNS name. + type: string + grpcServerTlsConfig: + description: 'GRPCServerTLSConfig configures the gRPC server from which + Thanos Querier reads recorded rule data. Note: Currently only the + CAFile, CertFile, and KeyFile fields are supported. Maps to the ''--grpc-server-tls-*'' + CLI args.' + properties: + ca: + description: Stuct containing the CA cert to use for the targets. + properties: + configMap: + description: ConfigMap containing data to use for the targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its key must + be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + type: object + caFile: + description: Path to the CA cert in the Prometheus container to + use for the targets. + type: string + cert: + description: Struct containing the client cert file for the targets. + properties: + configMap: + description: ConfigMap containing data to use for the targets. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its key must + be defined + type: boolean + required: + - key + type: object + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + type: object + certFile: + description: Path to the client cert file in the Prometheus container + for the targets. + type: string + insecureSkipVerify: + description: Disable target certificate validation. + type: boolean + keyFile: + description: Path to the client key file in the Prometheus container + for the targets. + type: string + keySecret: + description: Secret containing the client key file for the targets. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + serverName: + description: Used to verify the hostname for the targets. + type: string + type: object + image: + description: Thanos container image URL. + type: string + imagePullSecrets: + description: An optional list of references to secrets in the same namespace + to use for pulling thanos images from registries see http://kubernetes.io/docs/user-guide/images#specifying-imagepullsecrets-on-a-pod + items: + description: LocalObjectReference contains enough information to let + you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + type: array + initContainers: + description: 'InitContainers allows adding initContainers to the pod + definition. Those can be used to e.g. fetch secrets for injection + into the ThanosRuler configuration from external sources. Any errors + during the execution of an initContainer will lead to a restart of + the Pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/ + Using initContainers for any use case other then secret fetching is + entirely outside the scope of what the maintainers will support and + by doing so, you accept that this behaviour may break at any time + without notice.' + items: + description: A single application container that you want to run within + a pod. + properties: + args: + description: 'Arguments to the entrypoint. The docker image''s + CMD is used if this is not provided. Variable references $(VAR_NAME) + are expanded using the container''s environment. If a variable + cannot be resolved, the reference in the input string will be + unchanged. The $(VAR_NAME) syntax can be escaped with a double + $$, ie: $$(VAR_NAME). Escaped references will never be expanded, + regardless of whether the variable exists or not. Cannot be + updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within a shell. The + docker image''s ENTRYPOINT is used if this is not provided. + Variable references $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, the reference + in the input string will be unchanged. The $(VAR_NAME) syntax + can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable exists + or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previous defined environment variables in the + container and any service environment variables. If a + variable cannot be resolved, the reference in the input + string will be unchanged. The $(VAR_NAME) syntax can be + escaped with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable + exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + metadata.namespace, metadata.labels, metadata.annotations, + spec.nodeName, spec.serviceAccountName, status.hostIP, + status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, requests.memory + and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + description: Specifies the output format of the + exposed resources, defaults to "1" + type: string + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment variables + in the container. The keys defined within a source must be a + C_IDENTIFIER. All invalid keys will be reported as an event + when the container is starting. When a key exists in multiple + sources, the value associated with the last source will take + precedence. Values defined by an Env with a duplicate key will + take precedence. Cannot be updated. + items: + description: EnvFromSource represents the source of a set of + ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap must be defined + type: boolean + type: object + prefix: + description: An optional identifier to prepend to each key + in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret must be defined + type: boolean + type: object + type: object + type: array + image: + description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management + to default or override container images in workload controllers + like Deployments and StatefulSets.' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent + otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Actions that the management system should take in + response to container lifecycle events. Cannot be updated. + properties: + postStart: + description: 'PostStart is called immediately after a container + is created. If the handler fails, the container is terminated + and restarted according to its restart policy. Other management + of the container blocks until the hook completes. More info: + https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's filesystem. + The command is simply exec'd, it is not run inside + a shell, so traditional shell instructions ('|', + etc) won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is treated + as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately before a container + is terminated due to an API request or management event + such as liveness/startup probe failure, preemption, resource + contention, etc. The handler is not called if the container + crashes or exits. The reason for termination is passed to + the handler. The Pod''s termination grace period countdown + begins before the PreStop hooked is executed. Regardless + of the outcome of the handler, the container will eventually + terminate within the Pod''s termination grace period. Other + management of the container blocks until the hook completes + or until the termination grace period is reached. More info: + https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following should + be specified. Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute + inside the container, the working directory for + the command is root ('/') in the container's filesystem. + The command is simply exec'd, it is not run inside + a shell, so traditional shell instructions ('|', + etc) won't work. To use a shell, you need to explicitly + call out to that shell. Exit status of 0 is treated + as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to + the pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. + HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access + on the container. Number must be in the range 1 + to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + livenessProbe: + description: 'Periodic probe of container liveness. Container + will be restarted if the probe fails. Cannot be updated. More + info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + name: + description: Name of the container specified as a DNS_LABEL. Each + container in a pod must have a unique name (DNS_LABEL). Cannot + be updated. + type: string + ports: + description: List of ports to expose from the container. Exposing + a port here gives the system additional information about the + network connections a container uses, but is primarily informational. + Not specifying a port here DOES NOT prevent that port from being + exposed. Any port which is listening on the default "0.0.0.0" + address inside a container will be accessible from the network. + Cannot be updated. + items: + description: ContainerPort represents a network port in a single + container. + properties: + containerPort: + description: Number of port to expose on the pod's IP address. + This must be a valid port number, 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port to. + type: string + hostPort: + description: Number of port to expose on the host. If specified, + this must be a valid port number, 0 < x < 65536. If HostNetwork + is specified, this must match ContainerPort. Most containers + do not need this. + format: int32 + type: integer + name: + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port in a pod must + have a unique name. Name for the port that can be referred + to by services. + type: string + protocol: + description: Protocol for port. Must be UDP, TCP, or SCTP. + Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + readinessProbe: + description: 'Periodic probe of container service readiness. Container + will be removed from service endpoints if the probe fails. Cannot + be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + resources: + description: 'Compute Resources required by this container. Cannot + be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + properties: + limits: + additionalProperties: + type: string + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + type: string + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + securityContext: + description: 'Security options the pod should run with. More info: + https://kubernetes.io/docs/concepts/policy/security-context/ + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether a + process can gain more privileges than its parent process. + This bool directly controls if the no_new_privs flag will + be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as Privileged + 2) has CAP_SYS_ADMIN' + type: boolean + capabilities: + description: The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the + container runtime. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes in + privileged containers are essentially equivalent to root + on the host. Defaults to false. + type: boolean + procMount: + description: procMount denotes the type of proc mount to use + for the containers. The default is DefaultProcMount which + uses the container runtime defaults for readonly paths and + masked paths. This requires the ProcMountType feature flag + to be enabled. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only root filesystem. + Default is false. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the container + process. Uses runtime default if unset. May also be set + in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a non-root + user. If true, the Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 (root) and fail + to start the container if it does. If unset or false, no + such validation will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container + process. Defaults to user specified in image metadata if + unspecified. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random + SELinux context for each container. May also be set in + PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + windowsOptions: + description: The Windows specific settings applied to all + containers. If unspecified, the options from the PodSecurityContext + will be used. If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission + webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec named + by the GMSACredentialSpecName field. This field is alpha-level + and is only honored by servers that enable the WindowsGMSA + feature flag. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. This field is alpha-level + and is only honored by servers that enable the WindowsGMSA + feature flag. + type: string + runAsUserName: + description: The UserName in Windows to run the entrypoint + of the container process. Defaults to the user specified + in image metadata if unspecified. May also be set in + PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence. This field is beta-level and may be + disabled with the WindowsRunAsUserName feature flag. + type: string + type: object + type: object + startupProbe: + description: 'StartupProbe indicates that the Pod has successfully + initialized. If specified, no other probes are executed until + this completes successfully. If this probe fails, the Pod will + be restarted, just as if the livenessProbe failed. This can + be used to provide different probe parameters at the beginning + of a Pod''s lifecycle, when it might take a long time to load + data or warm a cache, than during steady-state operation. This + cannot be updated. This is an alpha feature enabled by the StartupProbe + feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + stdin: + description: Whether this container should allocate a buffer for + stdin in the container runtime. If this is not set, reads from + stdin in the container will always result in EOF. Default is + false. + type: boolean + stdinOnce: + description: Whether the container runtime should close the stdin + channel after it has been opened by a single attach. When stdin + is true the stdin stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin is opened on container + start, is empty until the first client attaches to stdin, and + then remains open and accepts data until the client disconnects, + at which time stdin is closed and remains closed until the container + is restarted. If this flag is false, a container processes that + reads from stdin will never receive an EOF. Default is false + type: boolean + terminationMessagePath: + description: 'Optional: Path at which the file to which the container''s + termination message will be written is mounted into the container''s + filesystem. Message written is intended to be brief final status, + such as an assertion failure message. Will be truncated by the + node if greater than 4096 bytes. The total message length across + all containers will be limited to 12kb. Defaults to /dev/termination-log. + Cannot be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message should be populated. + File will use the contents of terminationMessagePath to populate + the container status message on both success and failure. FallbackToLogsOnError + will use the last chunk of container log output if the termination + message file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, whichever + is smaller. Defaults to File. Cannot be updated. + type: string + tty: + description: Whether this container should allocate a TTY for + itself, also requires 'stdin' to be true. Default is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices to be + used by the container. This is a beta feature. + items: + description: volumeDevice describes a mapping of a raw block + device within a container. + properties: + devicePath: + description: devicePath is the path inside of the container + that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's filesystem. + Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: Path within the container at which the volume + should be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts are + propagated from the host to container and the other way + around. When not set, MountPropagationNone is used. This + field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the container's + volume should be mounted. Defaults to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from which + the container's volume should be mounted. Behaves similarly + to SubPath but environment variable references $(VAR_NAME) + are expanded using the container's environment. Defaults + to "" (volume's root). SubPathExpr and SubPath are mutually + exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If not specified, + the container runtime's default will be used, which might be + configured in the container image. Cannot be updated. + type: string + required: + - name + type: object + type: array + labels: + additionalProperties: + type: string + description: Labels configure the external label pairs to ThanosRuler. + If not provided, default replica label `thanos_ruler_replica` will + be added as a label and be dropped in alerts. + type: object + listenLocal: + description: ListenLocal makes the Thanos ruler listen on loopback, + so that it does not bind against the Pod IP. + type: boolean + logFormat: + description: Log format for ThanosRuler to be configured with. + type: string + logLevel: + description: Log level for ThanosRuler to be configured with. + type: string + nodeSelector: + additionalProperties: + type: string + description: Define which Nodes the Pods are scheduled on. + type: object + objectStorageConfig: + description: ObjectStorageConfig configures object storage in Thanos. + properties: + key: + description: The key of the secret to select from. Must be a valid + secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + paused: + description: When a ThanosRuler deployment is paused, no actions except + for deletion will be performed on the underlying objects. + type: boolean + podMetadata: + description: PodMetadata contains Labels and Annotations gets propagated + to the thanos ruler pods. + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value map stored + with a resource that may be set by external tools to store and + retrieve arbitrary metadata. They are not queryable and should + be preserved when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations' + type: object + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can be used to + organize and categorize (scope and select) objects. May match + selectors of replication controllers and services. More info: + http://kubernetes.io/docs/user-guide/labels' + type: object + type: object + portName: + description: Port name used for the pods and governing service. This + defaults to web + type: string + priorityClassName: + description: Priority class assigned to the Pods + type: string + queryConfig: + description: Define configuration for connecting to thanos query instances. + If this is defined, the QueryEndpoints field will be ignored. Maps + to the `query.config` CLI argument. Only available with thanos v0.11.0 + and higher. + properties: + key: + description: The key of the secret to select from. Must be a valid + secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + queryEndpoints: + description: QueryEndpoints defines Thanos querier endpoints from which + to query metrics. Maps to the --query flag of thanos ruler. + items: + type: string + type: array + replicas: + description: Number of thanos ruler instances to deploy. + format: int32 + type: integer + resources: + description: Resources defines the resource requirements for single + Pods. If not provided, no requests/limits will be set + properties: + limits: + additionalProperties: + type: string + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + type: string + description: 'Requests describes the minimum amount of compute resources + required. If Requests is omitted for a container, it defaults + to Limits if that is explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + retention: + description: Time duration ThanosRuler shall retain data for. Default + is '24h', and must match the regular expression `[0-9]+(ms|s|m|h|d|w|y)` + (milliseconds seconds minutes hours days weeks years). + type: string + routePrefix: + description: The route prefix ThanosRuler registers HTTP handlers for. + This allows thanos UI to be served on a sub-path. + type: string + ruleNamespaceSelector: + description: Namespaces to be selected for Rules discovery. If unspecified, + only the same namespace as the ThanosRuler object is in is used. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains + values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator + is In or NotIn, the values array must be non-empty. If the + operator is Exists or DoesNotExist, the values array must + be empty. This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator is + "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + ruleSelector: + description: A label selector to select which PrometheusRules to mount + for alerting and recording. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains + values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator + is In or NotIn, the values array must be non-empty. If the + operator is Exists or DoesNotExist, the values array must + be empty. This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator is + "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + securityContext: + description: SecurityContext holds pod-level security attributes and + common container settings. This defaults to the default PodSecurityContext. + properties: + fsGroup: + description: "A special supplemental group that applies to all containers + in a pod. Some volume types allow the Kubelet to change the ownership + of that volume to be owned by the pod: \n 1. The owning GID will + be the FSGroup 2. The setgid bit is set (new files created in + the volume will be owned by FSGroup) 3. The permission bits are + OR'd with rw-rw---- \n If unset, the Kubelet will not modify the + ownership and permissions of any volume." + format: int64 + type: integer + runAsGroup: + description: The GID to run the entrypoint of the container process. + Uses runtime default if unset. May also be set in SecurityContext. If + set in both SecurityContext and PodSecurityContext, the value + specified in SecurityContext takes precedence for that container. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a non-root + user. If true, the Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 (root) and fail to start + the container if it does. If unset or false, no such validation + will be performed. May also be set in SecurityContext. If set + in both SecurityContext and PodSecurityContext, the value specified + in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. May + also be set in SecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux + context for each container. May also be set in SecurityContext. If + set in both SecurityContext and PodSecurityContext, the value + specified in SecurityContext takes precedence for that container. + properties: + level: + description: Level is SELinux level label that applies to the + container. + type: string + role: + description: Role is a SELinux role label that applies to the + container. + type: string + type: + description: Type is a SELinux type label that applies to the + container. + type: string + user: + description: User is a SELinux user label that applies to the + container. + type: string + type: object + supplementalGroups: + description: A list of groups applied to the first process run in + each container, in addition to the container's primary GID. If + unspecified, no groups will be added to any container. + items: + format: int64 + type: integer + type: array + sysctls: + description: Sysctls hold a list of namespaced sysctls used for + the pod. Pods with unsupported sysctls (by the container runtime) + might fail to launch. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + description: The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext + will be used. If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission + webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec named by + the GMSACredentialSpecName field. This field is alpha-level + and is only honored by servers that enable the WindowsGMSA + feature flag. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA + credential spec to use. This field is alpha-level and is only + honored by servers that enable the WindowsGMSA feature flag. + type: string + runAsUserName: + description: The UserName in Windows to run the entrypoint of + the container process. Defaults to the user specified in image + metadata if unspecified. May also be set in PodSecurityContext. + If set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. This + field is beta-level and may be disabled with the WindowsRunAsUserName + feature flag. + type: string + type: object + type: object + serviceAccountName: + description: ServiceAccountName is the name of the ServiceAccount to + use to run the Thanos Ruler Pods. + type: string + storage: + description: Storage spec to specify how storage shall be used. + properties: + emptyDir: + description: 'EmptyDirVolumeSource to be used by the Prometheus + StatefulSets. If specified, used in place of any volumeClaimTemplate. + More info: https://kubernetes.io/docs/concepts/storage/volumes/#emptydir' + properties: + medium: + description: 'What type of storage medium should back this directory. + The default is "" which means to use the node''s default medium. + Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + description: 'Total amount of local storage required for this + EmptyDir volume. The size limit is also applicable for memory + medium. The maximum usage on memory medium EmptyDir would + be the minimum value between the SizeLimit specified here + and the sum of memory limits of all containers in a pod. The + default is nil which means that the limit is undefined. More + info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + type: string + type: object + volumeClaimTemplate: + description: A PVC spec to be used by the Prometheus StatefulSets. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this + representation of an object. Servers should convert recognized + schemas to the latest internal value, and may reject unrecognized + values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint + the client submits requests to. Cannot be updated. In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' + type: object + spec: + description: 'Spec defines the desired characteristics of a + volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'AccessModes contains the desired access modes + the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: This field requires the VolumeSnapshotDataSource + alpha feature gate to be enabled and currently VolumeSnapshot + is the only supported data source. If the provisioner + can support VolumeSnapshot data source, it will create + a new volume and data will be restored to the volume at + the same time. If the provisioner does not support VolumeSnapshot + data source, volume will not be created and the failure + will be reported as an event. In the future, we plan to + support more data source types and the behavior of the + provisioner may change. + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, the + specified Kind must be in the core API group. For + any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'Resources represents the minimum resources + the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + type: string + description: 'Limits describes the maximum amount of + compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + type: string + description: 'Requests describes the minimum amount + of compute resources required. If Requests is omitted + for a container, it defaults to Limits if that is + explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + selector: + description: A label query over volumes to consider for + binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists + or DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + storageClassName: + description: 'Name of the StorageClass required by the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume is required + by the claim. Value of Filesystem is implied when not + included in claim spec. This is a beta feature. + type: string + volumeName: + description: VolumeName is the binding reference to the + PersistentVolume backing this claim. + type: string + type: object + status: + description: 'Status represents the current information/status + of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'AccessModes contains the actual access modes + the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + capacity: + additionalProperties: + type: string + description: Represents the actual resources of the underlying + volume. + type: object + conditions: + description: Current Condition of persistent volume claim. + If underlying persistent volume is being resized then + the Condition will be set to 'ResizeStarted'. + items: + description: PersistentVolumeClaimCondition contails details + about state of pvc + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transitioned + from one status to another. + format: date-time + type: string + message: + description: Human-readable message indicating details + about last transition. + type: string + reason: + description: Unique, this should be a short, machine + understandable string that gives the reason for + condition's last transition. If it reports "ResizeStarted" + that means the underlying persistent volume is being + resized. + type: string + status: + type: string + type: + description: PersistentVolumeClaimConditionType is + a valid value of PersistentVolumeClaimCondition.Type + type: string + required: + - status + - type + type: object + type: array + phase: + description: Phase represents the current phase of PersistentVolumeClaim. + type: string + type: object + type: object + type: object + tolerations: + description: If specified, the pod's tolerations. + items: + description: The pod this Toleration is attached to tolerates any + taint that matches the triple using the matching + operator . + properties: + effect: + description: Effect indicates the taint effect to match. Empty + means match all taint effects. When specified, allowed values + are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration applies + to. Empty means match all taint keys. If the key is empty, operator + must be Exists; this combination means to match all values and + all keys. + type: string + operator: + description: Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. Exists + is equivalent to wildcard for value, so that a pod can tolerate + all taints of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of time the + toleration (which must be of effect NoExecute, otherwise this + field is ignored) tolerates the taint. By default, it is not + set, which means tolerate the taint forever (do not evict). + Zero and negative values will be treated as 0 (evict immediately) + by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise + just a regular string. + type: string + type: object + type: array + tracingConfig: + description: TracingConfig configures tracing in Thanos. This is an + experimental feature, it may change in any upcoming release in a breaking + way. + properties: + key: + description: The key of the secret to select from. Must be a valid + secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + volumes: + description: Volumes allows configuration of additional volumes on the + output StatefulSet definition. Volumes specified will be appended + to other volumes that are generated as a result of StorageSpec objects. + items: + description: Volume represents a named volume in a pod that may be + accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: 'AWSElasticBlockStore represents an AWS Disk resource + that is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + properties: + fsType: + description: 'Filesystem type of the volume that you want + to mount. Tip: Ensure that the filesystem type is supported + by the host operating system. Examples: "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem from compromising + the machine' + type: string + partition: + description: 'The partition in the volume that you want to + mount. If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition + as "1". Similarly, the volume partition for /dev/sda is + "0" (or you can leave the property empty).' + format: int32 + type: integer + readOnly: + description: 'Specify "true" to force and set the ReadOnly + property in VolumeMounts to "true". If omitted, the default + is "false". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'Unique ID of the persistent disk resource in + AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID + type: object + azureDisk: + description: AzureDisk represents an Azure Data Disk mount on + the host and bind mount to the pod. + properties: + cachingMode: + description: 'Host Caching mode: None, Read Only, Read Write.' + type: string + diskName: + description: The Name of the data disk in the blob storage + type: string + diskURI: + description: The URI the data disk in the blob storage + type: string + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + kind: + description: 'Expected values Shared: multiple blob disks + per storage account Dedicated: single blob disk per storage + account Managed: azure managed data disk (only in managed + availability set). defaults to shared' + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly here + will force the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: AzureFile represents an Azure File Service mount + on the host and bind mount to the pod. + properties: + readOnly: + description: Defaults to false (read/write). ReadOnly here + will force the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: the name of secret that contains Azure Storage + Account Name and Key + type: string + shareName: + description: Share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: CephFS represents a Ceph FS mount on the host that + shares a pod's lifetime + properties: + monitors: + description: 'Required: Monitors is a collection of Ceph monitors + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'Optional: Used as the mounted root, rather than + the full Ceph tree, default is /' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. More + info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'Optional: SecretFile is the path to key ring + for User, default is /etc/ceph/user.secret More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'Optional: SecretRef is reference to the authentication + secret for User, default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + user: + description: 'Optional: User is the rados user name, default + is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors + type: object + cinder: + description: 'Cinder represents a cinder volume attached and mounted + on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Examples: "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. More + info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'Optional: points to a secret object containing + parameters used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + volumeID: + description: 'volume id used to identify the volume in cinder. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + required: + - volumeID + type: object + configMap: + description: ConfigMap represents a configMap that should populate + this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created files + by default. Must be a value between 0 and 0777. Defaults + to 0644. Directories within the path are not affected by + this setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair in the Data + field of the referenced ConfigMap will be projected into + the volume as a file whose name is the key and content is + the value. If specified, the listed keys will be projected + into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the + ConfigMap, the volume setup will error unless it is marked + optional. Paths must be relative and may not contain the + '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to use on this file, + must be a value between 0 and 0777. If not specified, + the volume defaultMode will be used. This might be + in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode + bits set.' + format: int32 + type: integer + path: + description: The relative path of the file to map the + key to. May not be an absolute path. May not contain + the path element '..'. May not start with the string + '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or its keys must + be defined + type: boolean + type: object + csi: + description: CSI (Container Storage Interface) represents storage + that is handled by an external CSI driver (Alpha feature). + properties: + driver: + description: Driver is the name of the CSI driver that handles + this volume. Consult with your admin for the correct name + as registered in the cluster. + type: string + fsType: + description: Filesystem type to mount. Ex. "ext4", "xfs", + "ntfs". If not provided, the empty value is passed to the + associated CSI driver which will determine the default filesystem + to apply. + type: string + nodePublishSecretRef: + description: NodePublishSecretRef is a reference to the secret + object containing sensitive information to pass to the CSI + driver to complete the CSI NodePublishVolume and NodeUnpublishVolume + calls. This field is optional, and may be empty if no secret + is required. If the secret object contains more than one + secret, all secret references are passed. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + readOnly: + description: Specifies a read-only configuration for the volume. + Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: VolumeAttributes stores driver-specific properties + that are passed to the CSI driver. Consult your driver's + documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: DownwardAPI represents downward API about the pod + that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created files + by default. Must be a value between 0 and 0777. Defaults + to 0644. Directories within the path are not affected by + this setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: + only annotations, labels, name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits to use on this file, + must be a value between 0 and 0777. If not specified, + the volume defaultMode will be used. This might be + in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode + bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative path name + of the file to be created. Must not be absolute or + contain the ''..'' path. Must be utf-8 encoded. The + first item of the relative path must not start with + ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: only + resources limits and requests (limits.cpu, limits.memory, + requests.cpu and requests.memory) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + description: Specifies the output format of the + exposed resources, defaults to "1" + type: string + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + description: 'EmptyDir represents a temporary directory that shares + a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'What type of storage medium should back this + directory. The default is "" which means to use the node''s + default medium. Must be an empty string (default) or Memory. + More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + description: 'Total amount of local storage required for this + EmptyDir volume. The size limit is also applicable for memory + medium. The maximum usage on memory medium EmptyDir would + be the minimum value between the SizeLimit specified here + and the sum of memory limits of all containers in a pod. + The default is nil which means that the limit is undefined. + More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + type: string + type: object + fc: + description: FC represents a Fibre Channel resource that is attached + to a kubelet's host machine and then exposed to the pod. + properties: + fsType: + description: 'Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + TODO: how do we prevent errors in the filesystem from compromising + the machine' + type: string + lun: + description: 'Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'Optional: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts.' + type: boolean + targetWWNs: + description: 'Optional: FC target worldwide names (WWNs)' + items: + type: string + type: array + wwids: + description: 'Optional: FC volume world wide identifiers (wwids) + Either wwids or combination of targetWWNs and lun must be + set, but not both simultaneously.' + items: + type: string + type: array + type: object + flexVolume: + description: FlexVolume represents a generic volume resource that + is provisioned/attached using an exec based plugin. + properties: + driver: + description: Driver is the name of the driver to use for this + volume. + type: string + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". The default filesystem depends on FlexVolume + script. + type: string + options: + additionalProperties: + type: string + description: 'Optional: Extra command options if any.' + type: object + readOnly: + description: 'Optional: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts.' + type: boolean + secretRef: + description: 'Optional: SecretRef is reference to the secret + object containing sensitive information to pass to the plugin + scripts. This may be empty if no secret object is specified. + If the secret object contains more than one secret, all + secrets are passed to the plugin scripts.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + required: + - driver + type: object + flocker: + description: Flocker represents a Flocker volume attached to a + kubelet's host machine. This depends on the Flocker control + service being running + properties: + datasetName: + description: Name of the dataset stored as metadata -> name + on the dataset for Flocker should be considered as deprecated + type: string + datasetUUID: + description: UUID of the dataset. This is unique identifier + of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'GCEPersistentDisk represents a GCE Disk resource + that is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'Filesystem type of the volume that you want + to mount. Tip: Ensure that the filesystem type is supported + by the host operating system. Examples: "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem from compromising + the machine' + type: string + partition: + description: 'The partition in the volume that you want to + mount. If omitted, the default is to mount by volume name. + Examples: For volume /dev/sda1, you specify the partition + as "1". Similarly, the volume partition for /dev/sda is + "0" (or you can leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'Unique name of the PD resource in GCE. Used + to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'GitRepo represents a git repository at a particular + revision. DEPRECATED: GitRepo is deprecated. To provision a + container with a git repo, mount an EmptyDir into an InitContainer + that clones the repo using git, then mount the EmptyDir into + the Pod''s container.' + properties: + directory: + description: Target directory name. Must not contain or start + with '..'. If '.' is supplied, the volume directory will + be the git repository. Otherwise, if specified, the volume + will contain the git repository in the subdirectory with + the given name. + type: string + repository: + description: Repository URL + type: string + revision: + description: Commit hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'Glusterfs represents a Glusterfs mount on the host + that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'EndpointsName is the endpoint name that details + Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'Path is the Glusterfs volume path. More info: + https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'ReadOnly here will force the Glusterfs volume + to be mounted with read-only permissions. Defaults to false. + More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'HostPath represents a pre-existing file or directory + on the host machine that is directly exposed to the container. + This is generally used for system agents or other privileged + things that are allowed to see the host machine. Most containers + will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use host directory + mounts and who can/can not mount host directories as read/write.' + properties: + path: + description: 'Path of the directory on the host. If the path + is a symlink, it will follow the link to the real path. + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'Type for HostPath Volume Defaults to "" More + info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'ISCSI represents an ISCSI Disk resource that is + attached to a kubelet''s host machine and then exposed to the + pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: whether support iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: whether support iSCSI Session CHAP authentication + type: boolean + fsType: + description: 'Filesystem type of the volume that you want + to mount. Tip: Ensure that the filesystem type is supported + by the host operating system. Examples: "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem from compromising + the machine' + type: string + initiatorName: + description: Custom iSCSI Initiator Name. If initiatorName + is specified with iscsiInterface simultaneously, new iSCSI + interface : will be created + for the connection. + type: string + iqn: + description: Target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iSCSI Interface Name that uses an iSCSI transport. + Defaults to 'default' (tcp). + type: string + lun: + description: iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: iSCSI Target Portal List. The portal is either + an IP or ip_addr:port if the port is other than default + (typically TCP ports 860 and 3260). + items: + type: string + type: array + readOnly: + description: ReadOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: CHAP Secret for iSCSI target and initiator authentication + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + targetPortal: + description: iSCSI Target Portal. The Portal is either an + IP or ip_addr:port if the port is other than default (typically + TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'Volume''s name. Must be a DNS_LABEL and unique within + the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + nfs: + description: 'NFS represents an NFS mount on the host that shares + a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'Path that is exported by the NFS server. More + info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'ReadOnly here will force the NFS export to be + mounted with read-only permissions. Defaults to false. More + info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'Server is the hostname or IP address of the + NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'PersistentVolumeClaimVolumeSource represents a reference + to a PersistentVolumeClaim in the same namespace. More info: + https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + claimName: + description: 'ClaimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: PhotonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host machine + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + pdID: + description: ID that identifies Photon Controller persistent + disk + type: string + required: + - pdID + type: object + portworxVolume: + description: PortworxVolume represents a portworx volume attached + and mounted on kubelets host machine + properties: + fsType: + description: FSType represents the filesystem type to mount + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs". Implicitly inferred to be "ext4" + if unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly here + will force the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: VolumeID uniquely identifies a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: Items for all in one resources secrets, configmaps, + and downward API + properties: + defaultMode: + description: Mode bits to use on created files by default. + Must be a value between 0 and 0777. Directories within the + path are not affected by this setting. This might be in + conflict with other options that affect the file mode, like + fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + sources: + description: list of volume projections + items: + description: Projection that may be projected along with + other supported volume types + properties: + configMap: + description: information about the configMap data to + project + properties: + items: + description: If unspecified, each key-value pair + in the Data field of the referenced ConfigMap + will be projected into the volume as a file whose + name is the key and content is the value. If specified, + the listed keys will be projected into the specified + paths, and unlisted keys will not be present. + If a key is specified which is not present in + the ConfigMap, the volume setup will error unless + it is marked optional. Paths must be relative + and may not contain the '..' path or start with + '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to use on + this file, must be a value between 0 and + 0777. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the file + to map the key to. May not be an absolute + path. May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + keys must be defined + type: boolean + type: object + downwardAPI: + description: information about the downwardAPI data + to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing the + pod field + properties: + fieldRef: + description: 'Required: Selects a field of + the pod: only annotations, labels, name + and namespace are supported.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits to use on + this file, must be a value between 0 and + 0777. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must + not be absolute or contain the ''..'' path. + Must be utf-8 encoded. The first item of + the relative path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + description: Specifies the output format + of the exposed resources, defaults to + "1" + type: string + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + description: information about the secret data to project + properties: + items: + description: If unspecified, each key-value pair + in the Data field of the referenced Secret will + be projected into the volume as a file whose name + is the key and content is the value. If specified, + the listed keys will be projected into the specified + paths, and unlisted keys will not be present. + If a key is specified which is not present in + the Secret, the volume setup will error unless + it is marked optional. Paths must be relative + and may not contain the '..' path or start with + '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to use on + this file, must be a value between 0 and + 0777. If not specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the file + mode, like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the file + to map the key to. May not be an absolute + path. May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + type: object + serviceAccountToken: + description: information about the serviceAccountToken + data to project + properties: + audience: + description: Audience is the intended audience of + the token. A recipient of a token must identify + itself with an identifier specified in the audience + of the token, and otherwise should reject the + token. The audience defaults to the identifier + of the apiserver. + type: string + expirationSeconds: + description: ExpirationSeconds is the requested + duration of validity of the service account token. + As the token approaches expiration, the kubelet + volume plugin will proactively rotate the service + account token. The kubelet will start trying to + rotate the token if the token is older than 80 + percent of its time to live or if the token is + older than 24 hours.Defaults to 1 hour and must + be at least 10 minutes. + format: int64 + type: integer + path: + description: Path is the path relative to the mount + point of the file to project the token into. + type: string + required: + - path + type: object + type: object + type: array + required: + - sources + type: object + quobyte: + description: Quobyte represents a Quobyte mount on the host that + shares a pod's lifetime + properties: + group: + description: Group to map volume access to Default is no group + type: string + readOnly: + description: ReadOnly here will force the Quobyte volume to + be mounted with read-only permissions. Defaults to false. + type: boolean + registry: + description: Registry represents a single or multiple Quobyte + Registry services specified as a string as host:port pair + (multiple entries are separated with commas) which acts + as the central registry for volumes + type: string + tenant: + description: Tenant owning the given Quobyte volume in the + Backend Used with dynamically provisioned Quobyte volumes, + value is set by the plugin + type: string + user: + description: User to map volume access to Defaults to serivceaccount + user + type: string + volume: + description: Volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'RBD represents a Rados Block Device mount on the + host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' + properties: + fsType: + description: 'Filesystem type of the volume that you want + to mount. Tip: Ensure that the filesystem type is supported + by the host operating system. Examples: "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if unspecified. More info: + https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem from compromising + the machine' + type: string + image: + description: 'The rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'Keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'A collection of Ceph monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'The rados pool name. Default is rbd. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'SecretRef is name of the authentication secret + for RBDUser. If provided overrides keyring. Default is nil. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + user: + description: 'The rados user name. Default is admin. More + info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: ScaleIO represents a ScaleIO persistent volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Default is "xfs". + type: string + gateway: + description: The host address of the ScaleIO API Gateway. + type: string + protectionDomain: + description: The name of the ScaleIO Protection Domain for + the configured storage. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly here + will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef references to the secret for ScaleIO + user and other sensitive information. If this is not provided, + Login operation will fail. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + sslEnabled: + description: Flag to enable/disable SSL communication with + Gateway, default false + type: boolean + storageMode: + description: Indicates whether the storage for a volume should + be ThickProvisioned or ThinProvisioned. Default is ThinProvisioned. + type: string + storagePool: + description: The ScaleIO Storage Pool associated with the + protection domain. + type: string + system: + description: The name of the storage system as configured + in ScaleIO. + type: string + volumeName: + description: The name of a volume already created in the ScaleIO + system that is associated with this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'Secret represents a secret that should populate + this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'Optional: mode bits to use on created files + by default. Must be a value between 0 and 0777. Defaults + to 0644. Directories within the path are not affected by + this setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair in the Data + field of the referenced Secret will be projected into the + volume as a file whose name is the key and content is the + value. If specified, the listed keys will be projected into + the specified paths, and unlisted keys will not be present. + If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. + Paths must be relative and may not contain the '..' path + or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to use on this file, + must be a value between 0 and 0777. If not specified, + the volume defaultMode will be used. This might be + in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode + bits set.' + format: int32 + type: integer + path: + description: The relative path of the file to map the + key to. May not be an absolute path. May not contain + the path element '..'. May not start with the string + '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: Specify whether the Secret or its keys must be + defined + type: boolean + secretName: + description: 'Name of the secret in the pod''s namespace to + use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: StorageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly here + will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef specifies the secret to use for obtaining + the StorageOS API credentials. If not specified, default + values will be attempted. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + volumeName: + description: VolumeName is the human-readable name of the + StorageOS volume. Volume names are only unique within a + namespace. + type: string + volumeNamespace: + description: VolumeNamespace specifies the scope of the volume + within StorageOS. If no namespace is specified then the + Pod's namespace will be used. This allows the Kubernetes + name scoping to be mirrored within StorageOS for tighter + integration. Set VolumeName to any name to override the + default behaviour. Set to "default" if you are not using + namespaces within StorageOS. Namespaces that do not pre-exist + within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: VsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. "ext4", + "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + type: string + storagePolicyID: + description: Storage Policy Based Management (SPBM) profile + ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: Storage Policy Based Management (SPBM) profile + name. + type: string + volumePath: + description: Path that identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + type: object + status: + description: 'Most recent observed status of the ThanosRuler cluster. Read-only. + Not included when requesting from the apiserver, only from the ThanosRuler + Operator API itself. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' + properties: + availableReplicas: + description: Total number of available pods (ready for at least minReadySeconds) + targeted by this ThanosRuler deployment. + format: int32 + type: integer + paused: + description: Represents whether any actions on the underlying managed + objects are being performed. Only delete actions will be performed. + type: boolean + replicas: + description: Total number of non-terminated pods targeted by this ThanosRuler + deployment (their labels match the selector). + format: int32 + type: integer + unavailableReplicas: + description: Total number of unavailable pods targeted by this ThanosRuler + deployment. + format: int32 + type: integer + updatedReplicas: + description: Total number of non-terminated pods targeted by this ThanosRuler + deployment that have the desired version spec. + format: int32 + type: integer + required: + - availableReplicas + - paused + - replicas + - unavailableReplicas + - updatedReplicas + type: object + required: + - spec + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true diff --git a/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/templates/_helpers.tpl new file mode 100755 index 000000000..2da79e70f --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/templates/_helpers.tpl @@ -0,0 +1,29 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/templates/jobs.yaml b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/templates/jobs.yaml new file mode 100755 index 000000000..006c5ffff --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/templates/jobs.yaml @@ -0,0 +1,96 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-create + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }} + annotations: + "helm.sh/hook": post-install, post-upgrade, post-rollback + "helm.sh/hook-delete-policy": hook-succeeded +spec: + template: + metadata: + name: {{ .Chart.Name }}-create + labels: + app: {{ .Chart.Name }} + spec: + serviceAccountName: {{ .Chart.Name }}-manager + securityContext: + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: create-crds + image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/kubectl + - apply + - -f + - /etc/config/crd-manifest.yaml + volumeMounts: + - name: crd-manifest + readOnly: true + mountPath: /etc/config + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + volumes: + - name: crd-manifest + configMap: + name: {{ .Chart.Name }}-manifest +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-delete + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": hook-succeeded +spec: + template: + metadata: + name: {{ .Chart.Name }}-delete + labels: + app: {{ .Chart.Name }} + spec: + serviceAccountName: {{ .Chart.Name }}-manager + securityContext: + runAsNonRoot: true + runAsUser: 1000 + initContainers: + - name: remove-finalizers + image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/kubectl + - apply + - -f + - /etc/config/crd-manifest.yaml + volumeMounts: + - name: crd-manifest + readOnly: true + mountPath: /etc/config + containers: + - name: delete-crds + image: {{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/kubectl + - delete + - -f + - /etc/config/crd-manifest.yaml + volumeMounts: + - name: crd-manifest + readOnly: true + mountPath: /etc/config + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + volumes: + - name: crd-manifest + configMap: + name: {{ .Chart.Name }}-manifest diff --git a/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/templates/manifest.yaml b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/templates/manifest.yaml new file mode 100755 index 000000000..31016b6ef --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/templates/manifest.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Chart.Name }}-manifest + namespace: {{ .Release.Namespace }} +data: + crd-manifest.yaml: | + {{- $currentScope := . -}} + {{- $crds := (.Files.Glob "crd-manifest/**.yaml") -}} + {{- range $path, $_ := $crds -}} + {{- with $currentScope -}} + {{ .Files.Get $path | nindent 4 }} + --- + {{- end -}}{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/templates/rbac.yaml b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/templates/rbac.yaml new file mode 100755 index 000000000..bdda1ddad --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/templates/rbac.yaml @@ -0,0 +1,72 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }}-manager + labels: + app: {{ .Chart.Name }}-manager +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: ['create', 'get', 'patch', 'delete'] +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ .Chart.Name }}-manager +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-manager + labels: + app: {{ .Chart.Name }}-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-manager +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-manager +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ .Chart.Name }}-manager + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-manager +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'configMap' + - 'secret' diff --git a/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/values.yaml b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/values.yaml new file mode 100755 index 000000000..22a8a1c38 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring-crd/9.4.204-rc09/values.yaml @@ -0,0 +1,11 @@ +# Default values for rancher-monitoring-crd. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +global: + cattle: + systemDefaultRegistry: "" + +image: + repository: rancher/kubectl + tag: v1.20.2 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/.helmignore new file mode 100755 index 000000000..93bf1ec02 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/.helmignore @@ -0,0 +1,26 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +# helm/charts +OWNERS +hack/ +ci/ +kube-prometheus-*.tgz diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/CHANGELOG.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/CHANGELOG.md new file mode 100755 index 000000000..973b34dfb --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/CHANGELOG.md @@ -0,0 +1,47 @@ +# Changelog +All notable changes from the upstream Prometheus Operator chart will be added to this file. + +## [Package Version 00] - 2020-07-19 +### Added +- Added [Prometheus Adapter](https://github.com/helm/charts/tree/master/stable/prometheus-adapter) as a dependency to the upstream Prometheus Operator chart to allow users to expose custom metrics from the default Prometheus instance deployed by this chart +- Remove `prometheus-operator/cleanup-crds.yaml` and `prometheus-operator/crds.yaml` from the Prometheus Operator upstream chart in favor of just using the CRD directory to install the CRDs. +- Added support for `rkeControllerManager`, `rkeScheduler`, `rkeProxy`, and `rkeEtcd` PushProx exporters for monitoring k8s components within RKE clusters +- Added support for a `k3sServer` PushProx exporter that monitors k3s server components (`kubeControllerManager`, `kubeScheduler`, and `kubeProxy`) within k3s clusters +- Added support for `kubeAdmControllerManager`, `kubeAdmScheduler`, `kubeAdmProxy`, and `kubeAdmEtcd` PushProx exporters for monitoring k8s components within kubeAdm clusters +- Added support for `rke2ControllerManager`, `rke2Scheduler`, `rke2Proxy`, and `rke2Etcd` PushProx exporters for monitoring k8s components within rke2 clusters +- Exposed `prometheus.prometheusSpec.ignoreNamespaceSelectors` on values.yaml and set it to `false` by default. This value instructs the default Prometheus server deployed with this chart to ignore the `namespaceSelector` field within any created ServiceMonitor or PodMonitor CRs that it selects. This prevents ServiceMonitors and PodMonitors from configuring the Prometheus scrape configuration to monitor resources outside the namespace that they are deployed in; if a user needs to have one ServiceMonitor / PodMonitor monitor resources within several namespaces (such as the resources that are used to monitor Istio in a default installation), they should not enable this option since it would require them to create one ServiceMonitor / PodMonitor CR per namespace that they would like to monitor. Relevant fields were also updated in the default README.md. +- Added `grafana.sidecar.dashboards.searchNamespace` to `values.yaml` with a default value of `cattle-dashboards`. The namespace provided should contain all ConfigMaps with the label `grafana_dashboard` and will be searched by the Grafana Dashboards sidecar for updates. The namespace specified is also created along with this deployment. All default dashboard ConfigMaps have been relocated from the deployment namespace to the namespace specified +- Added `monitoring-admin`, `monitoring-edit`, and `monitoring-view` default `ClusterRoles` to allow admins to assign roles to users to interact with Prometheus Operator CRs. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`). In a typical RBAC setup, you might want to use a `ClusterRoleBinding` to bind these roles to a Subject to allow them to set up or view `ServiceMonitors` / `PodMonitors` / `PrometheusRules` and view `Prometheus` or `Alertmanager` CRs across the cluster. If `.Values.global.rbac.userRoles.aggregateRolesForRBAC` is enabled, these ClusterRoles will aggregate into the respective default ClusterRoles provided by Kubernetes +- Added `monitoring-config-admin`, `monitoring-config-edit` and `monitoring-config-view` default `Roles` to allow admins to assign roles to users to be able to edit / view `Secrets` and `ConfigMaps` within the `cattle-monitoring-system` namespace. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`). In a typical RBAC setup, you might want to use a `RoleBinding` to bind these roles to a Subject within the `cattle-monitoring-system` namespace to allow them to modify Secrets / ConfigMaps tied to the deployment, such as your Alertmanager Config Secret. +- Added `monitoring-dashboard-admin`, `monitoring-dashboard-edit` and `monitoring-dashboard-view` default `Roles` to allow admins to assign roles to users to be able to edit / view `ConfigMaps` within the `cattle-dashboards` namespace. These can be enabled by setting `.Values.global.rbac.userRoles.create` (default: `true`) and deploying Grafana as part of this chart. In a typical RBAC setup, you might want to use a `RoleBinding` to bind these roles to a Subject within the `cattle-dashboards` namespace to allow them to create / modify ConfigMaps that contain the JSON used to persist Grafana Dashboards on the cluster. +- Added default resource limits for `Prometheus Operator`, `Prometheus`, `AlertManager`, `Grafana`, `kube-state-metrics`, `node-exporter` +- Added a default template `rancher_defaults.tmpl` to AlertManager that Rancher will offer to users in order to help configure the way alerts are rendered on a notifier. Also updated the default template deployed with this chart to reference that template and added an example of a Slack config using this template as a comment in the `values.yaml`. +- Added support for private registries via introducing a new field for `global.cattle.systemDefaultRegistry` that, if supplied, will automatically be prepended onto every image used by the chart. +- Added a default `nginx` proxy container deployed with Grafana whose config is set in the `ConfigMap` located in `charts/grafana/templates/nginx-config.yaml`. The purpose of this container is to make it possible to view Grafana's UI through a proxy that has a subpath (e.g. Rancher's proxy). This proxy container is set to listen on port `8080` (with a `portName` of `nginx-http` instead of the default `service`), which is also where the Grafana service will now point to, and will forward all requests to the Grafana container listening on the default port `3000`. +- Added a default `nginx` proxy container deployed with Prometheus whose config is set in the `ConfigMap` located in `templates/prometheus/nginx-config.yaml`. The purpose of this container is to make it possible to view Prometheus's UI through a proxy that has a subpath (e.g. Rancher's proxy). This proxy container is set to listen on port `8080` (with a `portName` of `nginx-http` instead of the default `web`), which is also where the Prometheus service will now point to, and will forward all requests to the Prometheus container listening on the default port `9090`. +- Added support for passing CIS Scans in a hardened cluster by introducing a Job that patches the default service account within the `cattle-monitoring-system` and `cattle-dashboards` namespaces on install or upgrade and adding a default allow all `NetworkPolicy` to the `cattle-monitoring-system` and `cattle-dashboards` namespaces. +### Modified +- Updated the chart name from `prometheus-operator` to `rancher-monitoring` and added the `io.rancher.certified: rancher` annotation to `Chart.yaml` +- Modified the default `node-exporter` port from `9100` to `9796` +- Modified the default `nameOverride` to `rancher-monitoring`. This change is necessary as the Prometheus Adapter's default URL (`http://{{ .Values.nameOverride }}-prometheus.{{ .Values.namespaceOverride }}.svc`) is based off of the value used here; if modified, the default Adapter URL must also be modified +- Modified the default `namespaceOverride` to `cattle-monitoring-system`. This change is necessary as the Prometheus Adapter's default URL (`http://{{ .Values.nameOverride }}-prometheus.{{ .Values.namespaceOverride }}.svc`) is based off of the value used here; if modified, the default Adapter URL must also be modified +- Configured some default values for `grafana.service` values and exposed them in the default README.md +- The default namespaces the following ServiceMonitors were changed from the deployment namespace to allow them to continue to monitor metrics when `prometheus.prometheusSpec.ignoreNamespaceSelectors` is enabled: + - `core-dns`: `kube-system` + - `api-server`: `default` + - `kube-controller-manager`: `kube-system` + - `kubelet`: `{{ .Values.kubelet.namespace }}` +- Disabled the following deployments by default (can be enabled if required): + - `AlertManager` + - `kube-controller-manager` metrics exporter + - `kube-etcd` metrics exporter + - `kube-scheduler` metrics exporter + - `kube-proxy` metrics exporter +- Updated default Grafana `deploymentStrategy` to `Recreate` to prevent deployments from being stuck on upgrade if a PV is attached to Grafana +- Modified the default `SelectorNilUsesHelmValues` to default to `false`. As a result, we look for all CRs with any labels in all namespaces by default rather than just the ones tagged with the label `release: rancher-monitoring`. +- Modified the default images used by the `rancher-monitoring` chart to point to Rancher mirrors of the original images from upstream. +- Modified the behavior of the chart to create the Alertmanager Config Secret via a pre-install hook instead of using the normal Helm lifecycle to manage the secret. The benefit of this approach is that all changes to the Config Secret done on a live cluster will never get overridden on a `helm upgrade` since the secret only gets created on a `helm install`. If you would like the secret to be cleaned up on an `helm uninstall`, enable `alertmanager.cleanupOnUninstall`; however, this is disabled by default to prevent the loss of alerting configuration on an uninstall. This secret will never be modified on a `helm upgrade`. +- Modified the default `securityContext` for `Pod` templates across the chart to `{"runAsNonRoot": "true", "runAsUser": "1000"}` and replaced `grafana.rbac.pspUseAppArmor` in favor of `grafana.rbac.pspAnnotations={}` in order to make it possible to deploy this chart on a hardened cluster which does not support Seccomp or AppArmor annotations in PSPs. Users can always choose to specify the annotations they want to use for the PSP directly as part of the values provided. +- Modified `.Values.prometheus.prometheusSpec.containers` to take in a string representing a template that should be rendered by Helm (via `tpl`) instead of allowing a user to provide YAML directly. +- Modified the default Grafana configuration to auto assign users who access Grafana to the Viewer role and enable anonymous access to Grafana dashboards by default. This default works well for a Rancher user who is accessing Grafana via the `kubectl proxy` on the Rancher Dashboard UI since anonymous users who enter via the proxy are authenticated by the k8s API Server, but you can / should modify this behavior if you plan on exposing Grafana in a way that does not require authentication (e.g. as a `NodePort` service). +- Modified the default Grafana configuration to add a default dashboard for Rancher on the Grafana home page. \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/CONTRIBUTING.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/CONTRIBUTING.md new file mode 100755 index 000000000..f6ce2a323 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/CONTRIBUTING.md @@ -0,0 +1,12 @@ +# Contributing Guidelines + +## How to contribute to this chart + +1. Fork this repository, develop and test your Chart. +1. Bump the chart version for every change. +1. Ensure PR title has the prefix `[kube-prometheus-stack]` +1. When making changes to rules or dashboards, see the README.md section on how to sync data from upstream repositories +1. Check the `hack/minikube` folder has scripts to set up minikube and components of this chart that will allow all components to be scraped. You can use this configuration when validating your changes. +1. Check for changes of RBAC rules. +1. Check for changes in CRD specs. +1. PR must pass the linter (`helm lint`) diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/Chart.yaml new file mode 100755 index 000000000..a74deb899 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/Chart.yaml @@ -0,0 +1,46 @@ +annotations: + artifacthub.io/links: | + - name: Chart Source + url: https://github.com/prometheus-community/helm-charts + - name: Upstream Project + url: https://github.com/prometheus-operator/kube-prometheus + artifacthub.io/operator: "true" + catalog.cattle.io/auto-install: rancher-monitoring-crd=match + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: Monitoring + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/provides-gvr: monitoring.coreos.com.prometheus/v1 + catalog.cattle.io/release-name: rancher-monitoring + catalog.cattle.io/requests-cpu: 4500m + catalog.cattle.io/requests-memory: 4000Mi + catalog.cattle.io/ui-component: monitoring +apiVersion: v1 +appVersion: 0.38.1 +description: Collects several related Helm charts, Grafana dashboards, and Prometheus + rules combined with documentation and scripts to provide easy to operate end-to-end + Kubernetes cluster monitoring with Prometheus using the Prometheus Operator. +home: https://github.com/prometheus-operator/kube-prometheus +icon: https://raw.githubusercontent.com/prometheus/prometheus.github.io/master/assets/prometheus_logo-cb55bb5c346.png +keywords: +- operator +- prometheus +- kube-prometheus +- monitoring +maintainers: +- name: vsliouniaev +- name: bismarck +- email: gianrubio@gmail.com + name: gianrubio +- email: github.gkarthiks@gmail.com + name: gkarthiks +- email: scott@r6by.com + name: scottrigby +- email: miroslav.hadzhiev@gmail.com + name: Xtigyro +- email: arvind.iyengar@suse.com + name: Arvind +name: rancher-monitoring +sources: +- https://github.com/prometheus-community/helm-charts +- https://github.com/prometheus-operator/kube-prometheus +version: 9.4.204-rc09 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/README.md new file mode 100755 index 000000000..f95c12345 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/README.md @@ -0,0 +1,346 @@ +# kube-prometheus-stack + +Installs the [kube-prometheus stack](https://github.com/prometheus-operator/kube-prometheus), a collection of Kubernetes manifests, [Grafana](http://grafana.com/) dashboards, and [Prometheus rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with [Prometheus](https://prometheus.io/) using the [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). + +See the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) README for details about components, dashboards, and alerts. + +_Note: This chart was formerly named `prometheus-operator` chart, now renamed to more clearly reflect that it installs the `kube-prometheus` project stack, within which Prometheus Operator is only one component._ + +## Prerequisites + +- Kubernetes 1.10+ with Beta APIs +- Helm 2.12+ (If using Helm < 2.14, [see below for CRD workaround](#Helm-fails-to-create-CRDs)) + +## Get Repo Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo add stable https://charts.helm.sh/stable/ +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +# Helm 3 +$ helm install [RELEASE_NAME] prometheus-community/kube-prometheus-stack + +# Helm 2 +$ helm install --name [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Dependencies + +By default this chart installs additional, dependent charts: + +- [stable/kube-state-metrics](https://github.com/helm/charts/tree/master/stable/kube-state-metrics) +- [stable/prometheus-node-exporter](https://github.com/prometheus-community/helm-charts/tree/main/prometheus-node-exporter) +- [grafana/grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana) + +To disable dependencies during installation, see [multiple releases](#multiple-releases) below. + +_See [helm dependency](https://helm.sh/docs/helm/helm_dependency/) for command documentation._ + +## Uninstall Chart + +```console +# Helm 3 +$ helm uninstall [RELEASE_NAME] + +# Helm 2 +# helm delete --purge [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +CRDs created by this chart are not removed by default and should be manually cleaned up: + +```console +kubectl delete crd prometheuses.monitoring.coreos.com +kubectl delete crd prometheusrules.monitoring.coreos.com +kubectl delete crd servicemonitors.monitoring.coreos.com +kubectl delete crd podmonitors.monitoring.coreos.com +kubectl delete crd alertmanagers.monitoring.coreos.com +kubectl delete crd thanosrulers.monitoring.coreos.com +``` + +## Upgrading Chart + +```console +# Helm 3 or 2 +$ helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +### Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. + +### From 8.x to 9.x + +Version 9 of the helm chart removes the existing `additionalScrapeConfigsExternal` in favour of `additionalScrapeConfigsSecret`. This change lets users specify the secret name and secret key to use for the additional scrape configuration of prometheus. This is useful for users that have prometheus-operator as a subchart and also have a template that creates the additional scrape configuration. + +### From 7.x to 8.x + +Due to new template functions being used in the rules in version 8.x.x of the chart, an upgrade to Prometheus Operator and Prometheus is necessary in order to support them. First, upgrade to the latest version of 7.x.x + +```sh +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version 7.5.0 +``` + +Then upgrade to 8.x.x + +```sh +helm upgrade [RELEASE_NAME] prometheus-community/kube-prometheus-stack --version [8.x.x] +``` + +Minimal recommended Prometheus version for this chart release is `2.12.x` + +### From 6.x to 7.x + +Due to a change in grafana subchart, version 7.x.x now requires Helm >= 2.12.0. + +### From 5.x to 6.x + +Due to a change in deployment labels of kube-state-metrics, the upgrade requires `helm upgrade --force` in order to re-create the deployment. If this is not done an error will occur indicating that the deployment cannot be modified: + +```console +invalid: spec.selector: Invalid value: v1.LabelSelector{MatchLabels:map[string]string{"app.kubernetes.io/name":"kube-state-metrics"}, MatchExpressions:[]v1.LabelSelectorRequirement(nil)}: field is immutable +``` + +If this error has already been encountered, a `helm history` command can be used to determine which release has worked, then `helm rollback` to the release, then `helm upgrade --force` to this new one + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments: + +```console +helm show values prometheus-community/kube-prometheus-stack +``` + +You may also run `helm show values` on this chart's [dependencies](#dependencies) for additional options. + +### Rancher Monitoring Configuration + +The following table shows values exposed by Rancher Monitoring's additions to the chart: + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `nameOverride` | Provide a name that should be used instead of the chart name when naming all resources deployed by this chart |`"rancher-monitoring"`| +| `namespaceOverride` | Override the deployment namespace | `"cattle-monitoring-system"` | +| `global.rbac.userRoles.create` | Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets | `true` | +| `global.rbac.userRoles.aggregateToDefaultRoles` | Aggregate default user ClusterRoles into default k8s ClusterRoles | `true` | +| `prometheus-adapter.enabled` | Whether to install [prometheus-adapter](https://github.com/helm/charts/tree/master/stable/prometheus-adapter) within the cluster | `true` | +| `prometheus-adapter.prometheus.url` | A URL pointing to the Prometheus deployment within your cluster. The default value is set based on the assumption that you plan to deploy the default Prometheus instance from this chart where `.Values.namespaceOverride=cattle-monitoring-system` and `.Values.nameOverride=rancher-monitoring` | `http://rancher-monitoring-prometheus.cattle-monitoring-system.svc` | +| `prometheus-adapter.prometheus.port` | The port on the Prometheus deployment that Prometheus Adapter can make requests to | `9090` | +| `prometheus.prometheusSpec.ignoreNamespaceSelectors` | Ignore NamespaceSelector settings from the PodMonitor and ServiceMonitor configs. If true, PodMonitors and ServiceMonitors can only discover Pods and Services within the namespace they are deployed into | `false` | +| `alertmanager.secret.cleanupOnUninstall` | Whether or not to trigger a job to clean up the alertmanager config secret to be deleted on a `helm uninstall`. By default, this is disabled to prevent the loss of alerting configuration on an uninstall. | `false` | +| `alertmanager.secret.image.pullPolicy` | Image pull policy for job(s) related to alertmanager config secret's lifecycle | `IfNotPresent` | +| `alertmanager.secret.image.repository` | Repository to use for job(s) related to alertmanager config secret's lifecycle | `rancher/rancher-agent` | +| `alertmanager.secret.image.tag` | Tag to use for job(s) related to alertmanager config secret's lifecycle | `v2.4.8` | + +The following values are enabled for different distributions via [rancher-pushprox](https://github.com/rancher/dev-charts/tree/master/packages/rancher-pushprox). See the rancher-pushprox `README.md` for more information on what all values can be configured for the PushProxy chart. + +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `rkeControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in RKE clusters | `false` | +| `rkeScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in RKE clusters | `false` | +| `rkeProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in RKE clusters | `false` | +| `rkeEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in RKE clusters | `false` | +| `k3sServer.enabled` | Create a PushProx installation for monitoring k3s-server metrics (accounts for kube-controller-manager, kube-scheduler, and kube-proxy metrics) in k3s clusters | `false` | +| `kubeAdmControllerManager.enabled` | Create a PushProx installation for monitoring kube-controller-manager metrics in kubeAdm clusters | `false` | +| `kubeAdmScheduler.enabled` | Create a PushProx installation for monitoring kube-scheduler metrics in kubeAdm clusters | `false` | +| `kubeAdmProxy.enabled` | Create a PushProx installation for monitoring kube-proxy metrics in kubeAdm clusters | `false` | +| `kubeAdmEtcd.enabled` | Create a PushProx installation for monitoring etcd metrics in kubeAdm clusters | `false` | + + +### Multiple releases + +The same chart can be used to run multiple Prometheus instances in the same cluster if required. To achieve this, it is necessary to run only one instance of prometheus-operator and a pair of alertmanager pods for an HA configuration, while all other components need to be disabled. To disable a dependency during installation, set `kubeStateMetrics.enabled`, `nodeExporter.enabled` and `grafana.enabled` to `false`. + +## Work-Arounds for Known Issues + +### Running on private GKE clusters + +When Google configure the control plane for private clusters, they automatically configure VPC peering between your Kubernetes cluster’s network and a separate Google managed project. In order to restrict what Google are able to access within your cluster, the firewall rules configured restrict access to your Kubernetes pods. This means that in order to use the webhook component with a GKE private cluster, you must configure an additional firewall rule to allow the GKE control plane access to your webhook pod. + +You can read more information on how to add firewall rules for the GKE control plane nodes in the [GKE docs](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#add_firewall_rules) + +Alternatively, you can disable the hooks by setting `prometheusOperator.admissionWebhooks.enabled=false`. + +### Helm fails to create CRDs + +You should upgrade to Helm 2.14 + in order to avoid this issue. However, if you are stuck with an earlier Helm release you should instead use the following approach: Due to a bug in helm, it is possible for the 5 CRDs that are created by this chart to fail to get fully deployed before Helm attempts to create resources that require them. This affects all versions of Helm with a [potential fix pending](https://github.com/helm/helm/pull/5112). In order to work around this issue when installing the chart you will need to make sure all 5 CRDs exist in the cluster first and disable their previsioning by the chart: + +1. Create CRDs + + ```console + kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.38/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml + kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.38/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml + kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.38/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml + kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.38/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml + kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.38/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml + kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.38/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml + ``` + +2. Wait for CRDs to be created, which should only take a few seconds +3. [Install](#install-chart) the chart, but disable the CRD provisioning by setting `prometheusOperator.createCustomResource` to `false` + +## PrometheusRules Admission Webhooks + +With Prometheus Operator version 0.30+, the core Prometheus Operator pod exposes an endpoint that will integrate with the `validatingwebhookconfiguration` Kubernetes feature to prevent malformed rules from being added to the cluster. + +### How the Chart Configures the Hooks + +A validating and mutating webhook configuration requires the endpoint to which the request is sent to use TLS. It is possible to set up custom certificates to do this, but in most cases, a self-signed certificate is enough. The setup of this component requires some more complex orchestration when using helm. The steps are created to be idempotent and to allow turning the feature on and off without running into helm quirks. + +1. A pre-install hook provisions a certificate into the same namespace using a format compatible with provisioning using end-user certificates. If the certificate already exists, the hook exits. +2. The prometheus operator pod is configured to use a TLS proxy container, which will load that certificate. +3. Validating and Mutating webhook configurations are created in the cluster, with their failure mode set to Ignore. This allows rules to be created by the same chart at the same time, even though the webhook has not yet been fully set up - it does not have the correct CA field set. +4. A post-install hook reads the CA from the secret created by step 1 and patches the Validating and Mutating webhook configurations. This process will allow a custom CA provisioned by some other process to also be patched into the webhook configurations. The chosen failure policy is also patched into the webhook configurations + +### Alternatives + +It should be possible to use [jetstack/cert-manager](https://github.com/jetstack/cert-manager) if a more complete solution is required, but it has not been tested. + +### Limitations + +Because the operator can only run as a single pod, there is potential for this component failure to cause rule deployment failure. Because this risk is outweighed by the benefit of having validation, the feature is enabled by default. + +## Developing Prometheus Rules and Grafana Dashboards + +This chart Grafana Dashboards and Prometheus Rules are just a copy from [prometheus-operator/prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) and other sources, synced (with alterations) by scripts in [hack](hack) folder. In order to introduce any changes you need to first [add them to the original repo](https://github.com/prometheus-operator/kube-prometheus/blob/master/docs/developing-prometheus-rules-and-grafana-dashboards.md) and then sync there by scripts. + +## Further Information + +For more in-depth documentation of configuration options meanings, please see + +- [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) +- [Prometheus](https://prometheus.io/docs/introduction/overview/) +- [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana#grafana-helm-chart) + +## prometheus.io/scrape + +The prometheus operator does not support annotation-based discovery of services, using the `serviceMonitor` CRD in its place as it provides far more configuration options. For information on how to use servicemonitors, please see the documentation on the `prometheus-operator/prometheus-operator` documentation here: [Running Exporters](https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/user-guides/running-exporters.md) + +By default, Prometheus discovers ServiceMonitors within its namespace, that are labeled with the same release tag as the prometheus-operator release. Sometimes, you may need to discover custom ServiceMonitors, for example used to scrape data from third-party applications. An easy way of doing this, without compromising the default ServiceMonitors discovery, is allowing Prometheus to discover all ServiceMonitors within its namespace, without applying label filtering. To do so, you can set `prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues` to `false`. + +## Migrating from coreos/prometheus-operator chart + +The multiple charts have been combined into a single chart that installs prometheus operator, prometheus, alertmanager, grafana as well as the multitude of exporters necessary to monitor a cluster. + +There is no simple and direct migration path between the charts as the changes are extensive and intended to make the chart easier to support. + +The capabilities of the old chart are all available in the new chart, including the ability to run multiple prometheus instances on a single cluster - you will need to disable the parts of the chart you do not wish to deploy. + +You can check out the tickets for this change [here](https://github.com/prometheus-operator/prometheus-operator/issues/592) and [here](https://github.com/helm/charts/pull/6765). + +### High-level overview of Changes + +#### Added dependencies + +The chart has added 3 [dependencies](#dependencies). + +- Node-Exporter, Kube-State-Metrics: These components are loaded as dependencies into the chart, and are relatively simple components +- Grafana: The Grafana chart is more feature-rich than this chart - it contains a sidecar that is able to load data sources and dashboards from configmaps deployed into the same cluster. For more information check out the [documentation for the chart](https://github.com/helm/charts/tree/master/stable/grafana) + +#### CoreOS CRDs + +The CRDs are provisioned using a separate chart installation within the Helm chart `rancher-monitoring-crd` that is packaged alongside this chart. + +#### Kubelet Service + +Because the kubelet service has a new name in the chart, make sure to clean up the old kubelet service in the `kube-system` namespace to prevent counting container metrics twice. + +#### Persistent Volumes + +If you would like to keep the data of the current persistent volumes, it should be possible to attach existing volumes to new PVCs and PVs that are created using the conventions in the new chart. For example, in order to use an existing Azure disk for a helm release called `prometheus-migration` the following resources can be created: + +```yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvc-prometheus-migration-prometheus-0 +spec: + accessModes: + - ReadWriteOnce + azureDisk: + cachingMode: None + diskName: pvc-prometheus-migration-prometheus-0 + diskURI: /subscriptions/f5125d82-2622-4c50-8d25-3f7ba3e9ac4b/resourceGroups/sample-migration-resource-group/providers/Microsoft.Compute/disks/pvc-prometheus-migration-prometheus-0 + fsType: "" + kind: Managed + readOnly: false + capacity: + storage: 1Gi + persistentVolumeReclaimPolicy: Delete + storageClassName: prometheus + volumeMode: Filesystem +``` + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app: prometheus + prometheus: prometheus-migration-prometheus + name: prometheus-prometheus-migration-prometheus-db-prometheus-prometheus-migration-prometheus-0 + namespace: monitoring +spec: + accessModes: + - ReadWriteOnce + dataSource: null + resources: + requests: + storage: 1Gi + storageClassName: prometheus + volumeMode: Filesystem + volumeName: pvc-prometheus-migration-prometheus-0 +status: + accessModes: + - ReadWriteOnce + capacity: + storage: 1Gi +``` + +The PVC will take ownership of the PV and when you create a release using a persistent volume claim template it will use the existing PVCs as they match the naming convention used by the chart. For other cloud providers similar approaches can be used. + +#### KubeProxy + +The metrics bind address of kube-proxy is default to `127.0.0.1:10249` that prometheus instances **cannot** access to. You should expose metrics by changing `metricsBindAddress` field value to `0.0.0.0:10249` if you want to collect them. + +Depending on the cluster, the relevant part `config.conf` will be in ConfigMap `kube-system/kube-proxy` or `kube-system/kube-proxy-config`. For example: + +```console +kubectl -n kube-system edit cm kube-proxy +``` + +```yaml +apiVersion: v1 +data: + config.conf: |- + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + # ... + # metricsBindAddress: 127.0.0.1:10249 + metricsBindAddress: 0.0.0.0:10249 + # ... + kubeconfig.conf: |- + # ... +kind: ConfigMap +metadata: + labels: + app: kube-proxy + name: kube-proxy + namespace: kube-system +``` diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/app-README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/app-README.md new file mode 100755 index 000000000..af77e04ec --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/app-README.md @@ -0,0 +1,15 @@ +# Rancher Monitoring and Alerting + + This chart is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) chart. The chart deploys [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator) and its CRDs along with [Grafana](https://github.com/grafana/helm-charts/tree/main/charts/grafana), [Prometheus Adapter](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter) and additional charts / Kubernetes manifests to gather metrics. It allows users to monitor their Kubernetes clusters, view metrics in Grafana dashboards, and set up alerts and notifications. + +For more information on how to use the feature, refer to our [docs](https://rancher.com/docs/rancher/v2.x/en/monitoring-alerting/v2.5/). + +The chart installs the following components: + +- [Prometheus Operator](https://github.com/coreos/prometheus-operator) - The operator provides easy monitoring definitions for Kubernetes services, manages [Prometheus](https://prometheus.io/) and [AlertManager](https://prometheus.io/docs/alerting/latest/alertmanager/) instances, and adds default scrape targets for some Kubernetes components. +- [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) - A collection of community-curated Kubernetes manifests, Grafana Dashboards, and PrometheusRules that deploy a default end-to-end cluster monitoring configuration. +- [Grafana](https://github.com/helm/charts/tree/master/stable/grafana) - Grafana allows a user to create / view dashboards based on the cluster metrics collected by Prometheus. +- [node-exporter](https://github.com/helm/charts/tree/master/stable/prometheus-node-exporter) / [kube-state-metrics](https://github.com/helm/charts/tree/master/stable/kube-state-metrics) / [rancher-pushprox](https://github.com/rancher/charts/tree/dev-v2.5/packages/rancher-pushprox/charts) - These charts monitor various Kubernetes components across different Kubernetes cluster types. +- [Prometheus Adapter](https://github.com/helm/charts/tree/master/stable/prometheus-adapter) - The adapter allows a user to expose custom metrics, resource metrics, and external metrics on the default [Prometheus](https://prometheus.io/) instance to the Kubernetes API Server. + +For more information, review the Helm README of this chart. diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/.helmignore new file mode 100755 index 000000000..8cade1318 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.vscode +.project +.idea/ +*.tmproj +OWNERS diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/Chart.yaml new file mode 100755 index 000000000..8ab0187fb --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +appVersion: 7.1.5 +description: The leading tool for querying and visualizing time series and metrics. +home: https://grafana.net +icon: https://raw.githubusercontent.com/grafana/grafana/master/public/img/logo_transparent_400x.png +kubeVersion: ^1.8.0-0 +maintainers: +- email: zanhsieh@gmail.com + name: zanhsieh +- email: rluckie@cisco.com + name: rtluckie +- email: maor.friedman@redhat.com + name: maorfr +name: grafana +sources: +- https://github.com/grafana/grafana +version: 5.6.4 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/README.md new file mode 100755 index 000000000..b76c87406 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/README.md @@ -0,0 +1,424 @@ +# Grafana Helm Chart + +* Installs the web dashboarding system [Grafana](http://grafana.org/) + +## TL;DR + +```console +helm install grafana/grafana +``` + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```console +helm install --name my-release grafana/grafana +``` + +## Uninstalling the Chart + +To uninstall/delete the my-release deployment: + +```console +helm delete my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. + +## Upgrading an existing Release to a new major version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an +incompatible breaking change needing manual actions. + +### To 4.0.0 (And 3.12.1) + +This version requires Helm >= 2.12.0. + +### To 5.0.0 + +You have to add --force to your helm upgrade command as the labels of the chart have changed. + +## Configuration + +| Parameter | Description | Default | +|-------------------------------------------|-----------------------------------------------|---------------------------------------------------------| +| `replicas` | Number of nodes | `1` | +| `podDisruptionBudget.minAvailable` | Pod disruption minimum available | `nil` | +| `podDisruptionBudget.maxUnavailable` | Pod disruption maximum unavailable | `nil` | +| `deploymentStrategy` | Deployment strategy | `{ "type": "RollingUpdate" }` | +| `livenessProbe` | Liveness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } "initialDelaySeconds": 60, "timeoutSeconds": 30, "failureThreshold": 10 }` | +| `readinessProbe` | Readiness Probe settings | `{ "httpGet": { "path": "/api/health", "port": 3000 } }`| +| `securityContext` | Deployment securityContext | `{"runAsUser": 472, "runAsGroup": 472, "fsGroup": 472}` | +| `priorityClassName` | Name of Priority Class to assign pods | `nil` | +| `image.repository` | Image repository | `grafana/grafana` | +| `image.tag` | Image tag (`Must be >= 5.0.0`) | `7.0.3` | +| `image.sha` | Image sha (optional) | `17cbd08b9515fda889ca959e9d72ee6f3327c8f1844a3336dfd952134f38e2fe` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `image.pullSecrets` | Image pull secrets | `{}` | +| `service.type` | Kubernetes service type | `ClusterIP` | +| `service.port` | Kubernetes port where service is exposed | `80` | +| `service.portName` | Name of the port on the service | `service` | +| `service.targetPort` | Internal service is port | `3000` | +| `service.nodePort` | Kubernetes service nodePort | `nil` | +| `service.annotations` | Service annotations | `{}` | +| `service.labels` | Custom labels | `{}` | +| `service.clusterIP` | internal cluster service IP | `nil` | +| `service.loadBalancerIP` | IP address to assign to load balancer (if supported) | `nil` | +| `service.loadBalancerSourceRanges` | list of IP CIDRs allowed access to lb (if supported) | `[]` | +| `service.externalIPs` | service external IP addresses | `[]` | +| `extraExposePorts` | Additional service ports for sidecar containers| `[]` | +| `hostAliases` | adds rules to the pod's /etc/hosts | `[]` | +| `ingress.enabled` | Enables Ingress | `false` | +| `ingress.annotations` | Ingress annotations (values are templated) | `{}` | +| `ingress.labels` | Custom labels | `{}` | +| `ingress.path` | Ingress accepted path | `/` | +| `ingress.hosts` | Ingress accepted hostnames | `["chart-example.local"]` | +| `ingress.extraPaths` | Ingress extra paths to prepend to every host configuration. Useful when configuring [custom actions with AWS ALB Ingress Controller](https://kubernetes-sigs.github.io/aws-alb-ingress-controller/guide/ingress/annotation/#actions). | `[]` | +| `ingress.tls` | Ingress TLS configuration | `[]` | +| `resources` | CPU/Memory resource requests/limits | `{}` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `tolerations` | Toleration labels for pod assignment | `[]` | +| `affinity` | Affinity settings for pod assignment | `{}` | +| `extraInitContainers` | Init containers to add to the grafana pod | `{}` | +| `extraContainers` | Sidecar containers to add to the grafana pod | `{}` | +| `extraContainerVolumes` | Volumes that can be mounted in sidecar containers | `[]` | +| `schedulerName` | Name of the k8s scheduler (other than default) | `nil` | +| `persistence.enabled` | Use persistent volume to store data | `false` | +| `persistence.type` | Type of persistence (`pvc` or `statefulset`) | `pvc` | +| `persistence.size` | Size of persistent volume claim | `10Gi` | +| `persistence.existingClaim` | Use an existing PVC to persist data | `nil` | +| `persistence.storageClassName` | Type of persistent volume claim | `nil` | +| `persistence.accessModes` | Persistence access modes | `[ReadWriteOnce]` | +| `persistence.annotations` | PersistentVolumeClaim annotations | `{}` | +| `persistence.finalizers` | PersistentVolumeClaim finalizers | `[ "kubernetes.io/pvc-protection" ]` | +| `persistence.subPath` | Mount a sub dir of the persistent volume | `nil` | +| `initChownData.enabled` | If false, don't reset data ownership at startup | true | +| `initChownData.image.repository` | init-chown-data container image repository | `busybox` | +| `initChownData.image.tag` | init-chown-data container image tag | `1.31.1` | +| `initChownData.image.sha` | init-chown-data container image sha (optional)| `""` | +| `initChownData.image.pullPolicy` | init-chown-data container image pull policy | `IfNotPresent` | +| `initChownData.resources` | init-chown-data pod resource requests & limits | `{}` | +| `schedulerName` | Alternate scheduler name | `nil` | +| `env` | Extra environment variables passed to pods | `{}` | +| `envValueFrom` | Environment variables from alternate sources. See the API docs on [EnvVarSource](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core) for format details. | `{}` | +| `envFromSecret` | Name of a Kubernetes secret (must be manually created in the same namespace) containing values to be added to the environment. Can be templated | `""` | +| `envRenderSecret` | Sensible environment variables passed to pods and stored as secret | `{}` | +| `extraSecretMounts` | Additional grafana server secret mounts | `[]` | +| `extraVolumeMounts` | Additional grafana server volume mounts | `[]` | +| `extraConfigmapMounts` | Additional grafana server configMap volume mounts | `[]` | +| `extraEmptyDirMounts` | Additional grafana server emptyDir volume mounts | `[]` | +| `plugins` | Plugins to be loaded along with Grafana | `[]` | +| `datasources` | Configure grafana datasources (passed through tpl) | `{}` | +| `notifiers` | Configure grafana notifiers | `{}` | +| `dashboardProviders` | Configure grafana dashboard providers | `{}` | +| `dashboards` | Dashboards to import | `{}` | +| `dashboardsConfigMaps` | ConfigMaps reference that contains dashboards | `{}` | +| `grafana.ini` | Grafana's primary configuration | `{}` | +| `ldap.enabled` | Enable LDAP authentication | `false` | +| `ldap.existingSecret` | The name of an existing secret containing the `ldap.toml` file, this must have the key `ldap-toml`. | `""` | +| `ldap.config` | Grafana's LDAP configuration | `""` | +| `annotations` | Deployment annotations | `{}` | +| `labels` | Deployment labels | `{}` | +| `podAnnotations` | Pod annotations | `{}` | +| `podLabels` | Pod labels | `{}` | +| `podPortName` | Name of the grafana port on the pod | `grafana` | +| `sidecar.image.repository` | Sidecar image repository | `kiwigrid/k8s-sidecar` | +| `sidecar.image.tag` | Sidecar image tag | `0.1.151` | +| `sidecar.image.sha` | Sidecar image sha (optional) | `""` | +| `sidecar.imagePullPolicy` | Sidecar image pull policy | `IfNotPresent` | +| `sidecar.resources` | Sidecar resources | `{}` | +| `sidecar.enableUniqueFilenames` | Sets the kiwigrid/k8s-sidecar UNIQUE_FILENAMES environment variable | `false` | +| `sidecar.dashboards.enabled` | Enables the cluster wide search for dashboards and adds/updates/deletes them in grafana | `false` | +| `sidecar.dashboards.SCProvider` | Enables creation of sidecar provider | `true` | +| `sidecar.dashboards.provider.name` | Unique name of the grafana provider | `sidecarProvider` | +| `sidecar.dashboards.provider.orgid` | Id of the organisation, to which the dashboards should be added | `1` | +| `sidecar.dashboards.provider.folder` | Logical folder in which grafana groups dashboards | `""` | +| `sidecar.dashboards.provider.disableDelete` | Activate to avoid the deletion of imported dashboards | `false` | +| `sidecar.dashboards.provider.allowUiUpdates` | Allow updating provisioned dashboards from the UI | `false` | +| `sidecar.dashboards.provider.type` | Provider type | `file` | +| `sidecar.dashboards.watchMethod` | Method to use to detect ConfigMap changes. With WATCH the sidecar will do a WATCH requests, with SLEEP it will list all ConfigMaps, then sleep for 60 seconds. | `WATCH` | +| `sidecar.skipTlsVerify` | Set to true to skip tls verification for kube api calls | `nil` | +| `sidecar.dashboards.label` | Label that config maps with dashboards should have to be added | `grafana_dashboard` | +| `sidecar.dashboards.folder` | Folder in the pod that should hold the collected dashboards (unless `sidecar.dashboards.defaultFolderName` is set). This path will be mounted. | `/tmp/dashboards` | +| `sidecar.dashboards.defaultFolderName` | The default folder name, it will create a subfolder under the `sidecar.dashboards.folder` and put dashboards in there instead | `nil` | +| `sidecar.dashboards.searchNamespace` | If specified, the sidecar will search for dashboard config-maps inside this namespace. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces | `nil` | +| `sidecar.datasources.enabled` | Enables the cluster wide search for datasources and adds/updates/deletes them in grafana |`false` | +| `sidecar.datasources.label` | Label that config maps with datasources should have to be added | `grafana_datasource` | +| `sidecar.datasources.searchNamespace` | If specified, the sidecar will search for datasources config-maps inside this namespace. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces | `nil` | +| `sidecar.notifiers.enabled` | Enables the cluster wide search for notifiers and adds/updates/deletes them in grafana |`false` | +| `sidecar.notifiers.label` | Label that config maps with notifiers should have to be added | `grafana_notifier` | +| `sidecar.notifiers.searchNamespace` | If specified, the sidecar will search for notifiers config-maps (or secrets) inside this namespace. Otherwise the namespace in which the sidecar is running will be used. It's also possible to specify ALL to search in all namespaces | `nil` | +| `smtp.existingSecret` | The name of an existing secret containing the SMTP credentials. | `""` | +| `smtp.userKey` | The key in the existing SMTP secret containing the username. | `"user"` | +| `smtp.passwordKey` | The key in the existing SMTP secret containing the password. | `"password"` | +| `admin.existingSecret` | The name of an existing secret containing the admin credentials. | `""` | +| `admin.userKey` | The key in the existing admin secret containing the username. | `"admin-user"` | +| `admin.passwordKey` | The key in the existing admin secret containing the password. | `"admin-password"` | +| `serviceAccount.annotations` | ServiceAccount annotations | | +| `serviceAccount.create` | Create service account | `true` | +| `serviceAccount.name` | Service account name to use, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `` | +| `serviceAccount.nameTest` | Service account name to use for test, when empty will be set to created account if `serviceAccount.create` is set else to `default` | `nil` | +| `rbac.create` | Create and use RBAC resources | `true` | +| `rbac.namespaced` | Creates Role and Rolebinding instead of the default ClusterRole and ClusteRoleBindings for the grafana instance | `false` | +| `rbac.pspEnabled` | Create PodSecurityPolicy (with `rbac.create`, grant roles permissions as well) | `true` | +| `rbac.pspUseAppArmor` | Enforce AppArmor in created PodSecurityPolicy (requires `rbac.pspEnabled`) | `true` | +| `rbac.extraRoleRules` | Additional rules to add to the Role | [] | +| `rbac.extraClusterRoleRules` | Additional rules to add to the ClusterRole | [] | +| `command` | Define command to be executed by grafana container at startup | `nil` | +| `testFramework.enabled` | Whether to create test-related resources | `true` | +| `testFramework.image` | `test-framework` image repository. | `bats/bats` | +| `testFramework.tag` | `test-framework` image tag. | `v1.1.0` | +| `testFramework.imagePullPolicy` | `test-framework` image pull policy. | `IfNotPresent` | +| `testFramework.securityContext` | `test-framework` securityContext | `{}` | +| `downloadDashboards.env` | Environment variables to be passed to the `download-dashboards` container | `{}` | +| `downloadDashboards.resources` | Resources of `download-dashboards` container | `{}` | +| `downloadDashboardsImage.repository` | Curl docker image repo | `curlimages/curl` | +| `downloadDashboardsImage.tag` | Curl docker image tag | `7.70.0` | +| `downloadDashboardsImage.sha` | Curl docker image sha (optional) | `""` | +| `downloadDashboardsImage.pullPolicy` | Curl docker image pull policy | `IfNotPresent` | +| `namespaceOverride` | Override the deployment namespace | `""` (`Release.Namespace`) | +| `serviceMonitor.enabled` | Use servicemonitor from prometheus operator | `false` | +| `serviceMonitor.namespace` | Namespace this servicemonitor is installed in | | +| `serviceMonitor.interval` | How frequently Prometheus should scrape | `1m` | +| `serviceMonitor.path` | Path to scrape | `/metrics` | +| `serviceMonitor.labels` | Labels for the servicemonitor passed to Prometheus Operator | `{}` | +| `serviceMonitor.scrapeTimeout` | Timeout after which the scrape is ended | `30s` | +| `serviceMonitor.relabelings` | MetricRelabelConfigs to apply to samples before ingestion. | `[]` | + +### Example ingress with path + +With grafana 6.3 and above +```yaml +grafana.ini: + server: + domain: monitoring.example.com + root_url: "%(protocol)s://%(domain)s/grafana" + serve_from_sub_path: true +ingress: + enabled: true + hosts: + - "monitoring.example.com" + path: "/grafana" +``` + +### Example of extraVolumeMounts + +```yaml +- extraVolumeMounts: + - name: plugins + mountPath: /var/lib/grafana/plugins + subPath: configs/grafana/plugins + existingClaim: existing-grafana-claim + readOnly: false +``` + +## Import dashboards + +There are a few methods to import dashboards to Grafana. Below are some examples and explanations as to how to use each method: + +```yaml +dashboards: + default: + some-dashboard: + json: | + { + "annotations": + + ... + # Complete json file here + ... + + "title": "Some Dashboard", + "uid": "abcd1234", + "version": 1 + } + custom-dashboard: + # This is a path to a file inside the dashboards directory inside the chart directory + file: dashboards/custom-dashboard.json + prometheus-stats: + # Ref: https://grafana.com/dashboards/2 + gnetId: 2 + revision: 2 + datasource: Prometheus + local-dashboard: + url: https://raw.githubusercontent.com/user/repository/master/dashboards/dashboard.json +``` + +## BASE64 dashboards + +Dashboards could be stored on a server that does not return JSON directly and instead of it returns a Base64 encoded file (e.g. Gerrit) +A new parameter has been added to the url use case so if you specify a b64content value equals to true after the url entry a Base64 decoding is applied before save the file to disk. +If this entry is not set or is equals to false not decoding is applied to the file before saving it to disk. + +### Gerrit use case + +Gerrit API for download files has the following schema: where {project-name} and +{file-id} usually has '/' in their values and so they MUST be replaced by %2F so if project-name is user/repo, branch-id is master and file-id is equals to dir1/dir2/dashboard +the url value is + +## Sidecar for dashboards + +If the parameter `sidecar.dashboards.enabled` is set, a sidecar container is deployed in the grafana +pod. This container watches all configmaps (or secrets) in the cluster and filters out the ones with +a label as defined in `sidecar.dashboards.label`. The files defined in those configmaps are written +to a folder and accessed by grafana. Changes to the configmaps are monitored and the imported +dashboards are deleted/updated. + +A recommendation is to use one configmap per dashboard, as a reduction of multiple dashboards inside +one configmap is currently not properly mirrored in grafana. + +Example dashboard config: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: sample-grafana-dashboard + labels: + grafana_dashboard: "1" +data: + k8s-dashboard.json: |- + [...] +``` + +## Sidecar for datasources + +If the parameter `sidecar.datasources.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.datasources.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the data sources in grafana can be imported. The secrets must be created before `helm install` so +that the datasources init container can list the secrets. + +Secrets are recommended over configmaps for this usecase because datasources usually contain private +data like usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example datasource config adapted from [Grafana](http://docs.grafana.org/administration/provisioning/#example-datasource-config-file): + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: sample-grafana-datasource + labels: + grafana_datasource: "1" +type: Opaque +stringData: + datasource.yaml: |- + # config file version + apiVersion: 1 + + # list of datasources that should be deleted from the database + deleteDatasources: + - name: Graphite + orgId: 1 + + # list of datasources to insert/update depending + # whats available in the database + datasources: + # name of the datasource. Required + - name: Graphite + # datasource type. Required + type: graphite + # access mode. proxy or direct (Server or Browser in the UI). Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://localhost:8080 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basicAuth: + # basic auth username + basicAuthUser: + # basic auth password + basicAuthPassword: + # enable/disable with credentials headers + withCredentials: + # mark as default datasource. Max one per org + isDefault: + # fields that will be converted to json and stored in json_data + jsonData: + graphiteVersion: "1.1" + tlsAuth: true + tlsAuthWithCACert: true + # json object of data that will be encrypted. + secureJsonData: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: false + +``` + +## Sidecar for notifiers + +If the parameter `sidecar.notifiers.enabled` is set, an init container is deployed in the grafana +pod. This container lists all secrets (or configmaps, though not recommended) in the cluster and +filters out the ones with a label as defined in `sidecar.notifiers.label`. The files defined in +those secrets are written to a folder and accessed by grafana on startup. Using these yaml files, +the notification channels in grafana can be imported. The secrets must be created before +`helm install` so that the notifiers init container can list the secrets. + +Secrets are recommended over configmaps for this usecase because alert notification channels usually contain +private data like SMTP usernames and passwords. Secrets are the more appropriate cluster resource to manage those. + +Example datasource config adapted from [Grafana](https://grafana.com/docs/grafana/latest/administration/provisioning/#alert-notification-channels): + +```yaml +notifiers: + - name: notification-channel-1 + type: slack + uid: notifier1 + # either + org_id: 2 + # or + org_name: Main Org. + is_default: true + send_reminder: true + frequency: 1h + disable_resolve_message: false + # See `Supported Settings` section for settings supporter for each + # alert notification type. + settings: + recipient: 'XXX' + token: 'xoxb' + uploadImage: true + url: https://slack.com + +delete_notifiers: + - name: notification-channel-1 + uid: notifier1 + org_id: 2 + - name: notification-channel-2 + # default org_id: 1 +``` + +## How to serve Grafana with a path prefix (/grafana) + +In order to serve Grafana with a prefix (e.g., ), add the following to your values.yaml. + +```yaml +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/use-regex: "true" + + path: /grafana/?(.*) + hosts: + - k8s.example.dev + +grafana.ini: + server: + root_url: http://localhost:3000/grafana # this host can be localhost +``` diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/dashboards/custom-dashboard.json b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/dashboards/custom-dashboard.json new file mode 100755 index 000000000..9e26dfeeb --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/dashboards/custom-dashboard.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/NOTES.txt b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/NOTES.txt new file mode 100755 index 000000000..1fc8436d9 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/NOTES.txt @@ -0,0 +1,54 @@ +1. Get your '{{ .Values.adminUser }}' user password by running: + + kubectl get secret --namespace {{ template "grafana.namespace" . }} {{ template "grafana.fullname" . }} -o jsonpath="{.data.admin-password}" | base64 --decode ; echo + +2. The Grafana server can be accessed via port {{ .Values.service.port }} on the following DNS name from within your cluster: + + {{ template "grafana.fullname" . }}.{{ template "grafana.namespace" . }}.svc.cluster.local +{{ if .Values.ingress.enabled }} + If you bind grafana to 80, please update values in values.yaml and reinstall: + ``` + securityContext: + runAsUser: 0 + runAsGroup: 0 + fsGroup: 0 + + command: + - "setcap" + - "'cap_net_bind_service=+ep'" + - "/usr/sbin/grafana-server &&" + - "sh" + - "/run.sh" + ``` + Details refer to https://grafana.com/docs/installation/configuration/#http-port. + Or grafana would always crash. + + From outside the cluster, the server URL(s) are: +{{- range .Values.ingress.hosts }} + http://{{ . }} +{{- end }} +{{ else }} + Get the Grafana URL to visit by running these commands in the same shell: +{{ if contains "NodePort" .Values.service.type -}} + export NODE_PORT=$(kubectl get --namespace {{ template "grafana.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "grafana.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ template "grafana.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{ else if contains "LoadBalancer" .Values.service.type -}} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc --namespace {{ template "grafana.namespace" . }} -w {{ template "grafana.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ template "grafana.namespace" . }} {{ template "grafana.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + http://$SERVICE_IP:{{ .Values.service.port -}} +{{ else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ template "grafana.namespace" . }} -l "app.kubernetes.io/name={{ template "grafana.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + kubectl --namespace {{ template "grafana.namespace" . }} port-forward $POD_NAME 3000 +{{- end }} +{{- end }} + +3. Login with the password from step 1 and the username: {{ .Values.adminUser }} + +{{- if not .Values.persistence.enabled }} +################################################################################# +###### WARNING: Persistence is disabled!!! You will lose your data when ##### +###### the Grafana pod is terminated. ##### +################################################################################# +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/_helpers.tpl new file mode 100755 index 000000000..cbe4f6086 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/_helpers.tpl @@ -0,0 +1,82 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "grafana.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "grafana.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "grafana.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the name of the service account +*/}} +{{- define "grafana.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "grafana.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{- define "grafana.serviceAccountNameTest" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (print (include "grafana.fullname" .) "-test") .Values.serviceAccount.nameTest }} +{{- else -}} + {{ default "default" .Values.serviceAccount.nameTest }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "grafana.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "grafana.labels" -}} +helm.sh/chart: {{ include "grafana.chart" . }} +{{ include "grafana.selectorLabels" . }} +{{- if or .Chart.AppVersion .Values.image.tag }} +app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "grafana.selectorLabels" -}} +app.kubernetes.io/name: {{ include "grafana.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/_pod.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/_pod.tpl new file mode 100755 index 000000000..ee27e1bfc --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/_pod.tpl @@ -0,0 +1,448 @@ +{{- define "grafana.pod" -}} +{{- if .Values.schedulerName }} +schedulerName: "{{ .Values.schedulerName }}" +{{- end }} +serviceAccountName: {{ template "grafana.serviceAccountName" . }} +{{- if .Values.securityContext }} +securityContext: +{{ toYaml .Values.securityContext | indent 2 }} +{{- end }} +{{- if .Values.hostAliases }} +hostAliases: +{{ toYaml .Values.hostAliases | indent 2 }} +{{- end }} +{{- if .Values.priorityClassName }} +priorityClassName: {{ .Values.priorityClassName }} +{{- end }} +{{- if ( or .Values.persistence.enabled .Values.dashboards .Values.sidecar.datasources.enabled .Values.sidecar.notifiers.enabled .Values.extraInitContainers) }} +initContainers: +{{- end }} +{{- if ( and .Values.persistence.enabled .Values.initChownData.enabled ) }} + - name: init-chown-data + {{- if .Values.initChownData.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}@sha256:{{ .Values.initChownData.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.initChownData.image.repository }}:{{ .Values.initChownData.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.initChownData.image.pullPolicy }} + securityContext: + runAsNonRoot: false + runAsUser: 0 + command: ["chown", "-R", "{{ .Values.securityContext.runAsUser }}:{{ .Values.securityContext.runAsGroup }}", "/var/lib/grafana"] + resources: +{{ toYaml .Values.initChownData.resources | indent 6 }} + volumeMounts: + - name: storage + mountPath: "/var/lib/grafana" +{{- if .Values.persistence.subPath }} + subPath: {{ .Values.persistence.subPath }} +{{- end }} +{{- end }} +{{- if .Values.dashboards }} + - name: download-dashboards + {{- if .Values.downloadDashboardsImage.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}@sha256:{{ .Values.downloadDashboardsImage.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.downloadDashboardsImage.repository }}:{{ .Values.downloadDashboardsImage.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.downloadDashboardsImage.pullPolicy }} + command: ["/bin/sh"] + args: [ "-c", "mkdir -p /var/lib/grafana/dashboards/default && /bin/sh /etc/grafana/download_dashboards.sh" ] + resources: +{{ toYaml .Values.downloadDashboards.resources | indent 6 }} + env: +{{- range $key, $value := .Values.downloadDashboards.env }} + - name: "{{ $key }}" + value: "{{ $value }}" +{{- end }} + volumeMounts: + - name: config + mountPath: "/etc/grafana/download_dashboards.sh" + subPath: download_dashboards.sh + - name: storage + mountPath: "/var/lib/grafana" +{{- if .Values.persistence.subPath }} + subPath: {{ .Values.persistence.subPath }} +{{- end }} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + {{- end }} +{{- end }} +{{- if .Values.sidecar.datasources.enabled }} + - name: {{ template "grafana.name" . }}-sc-datasources + {{- if .Values.sidecar.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + - name: METHOD + value: LIST + - name: LABEL + value: "{{ .Values.sidecar.datasources.label }}" + - name: FOLDER + value: "/etc/grafana/provisioning/datasources" + - name: RESOURCE + value: "both" + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- if .Values.sidecar.datasources.searchNamespace }} + - name: NAMESPACE + value: "{{ .Values.sidecar.datasources.searchNamespace }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + resources: +{{ toYaml .Values.sidecar.resources | indent 6 }} + volumeMounts: + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end}} +{{- if .Values.sidecar.notifiers.enabled }} + - name: {{ template "grafana.name" . }}-sc-notifiers + {{- if .Values.sidecar.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + - name: METHOD + value: LIST + - name: LABEL + value: "{{ .Values.sidecar.notifiers.label }}" + - name: FOLDER + value: "/etc/grafana/provisioning/notifiers" + - name: RESOURCE + value: "both" + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- if .Values.sidecar.notifiers.searchNamespace }} + - name: NAMESPACE + value: "{{ .Values.sidecar.notifiers.searchNamespace }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + resources: +{{ toYaml .Values.sidecar.resources | indent 6 }} + volumeMounts: + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} +{{- if .Values.extraInitContainers }} +{{ toYaml .Values.extraInitContainers | indent 2 }} +{{- end }} +{{- if .Values.image.pullSecrets }} +imagePullSecrets: +{{- range .Values.image.pullSecrets }} + - name: {{ . }} +{{- end}} +{{- end }} +containers: +{{- if .Values.sidecar.dashboards.enabled }} + - name: {{ template "grafana.name" . }}-sc-dashboard + {{- if .Values.sidecar.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.imagePullPolicy }} + env: + - name: METHOD + value: {{ .Values.sidecar.dashboards.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.dashboards.label }}" + - name: FOLDER + value: "{{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }}" + - name: RESOURCE + value: "both" + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- if .Values.sidecar.dashboards.searchNamespace }} + - name: NAMESPACE + value: "{{ .Values.sidecar.dashboards.searchNamespace }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + resources: +{{ toYaml .Values.sidecar.resources | indent 6 }} + volumeMounts: + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} +{{- end}} + - name: {{ .Chart.Name }} + {{- if .Values.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}@sha256:{{ .Values.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.command }} + command: + {{- range .Values.command }} + - {{ . }} + {{- end }} + {{- end}} + volumeMounts: + - name: config + mountPath: "/etc/grafana/grafana.ini" + subPath: grafana.ini + {{- if .Values.ldap.enabled }} + - name: ldap + mountPath: "/etc/grafana/ldap.toml" + subPath: ldap.toml + {{- end }} + {{- range .Values.extraConfigmapMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + - name: storage + mountPath: "/var/lib/grafana" +{{- if .Values.persistence.subPath }} + subPath: {{ .Values.persistence.subPath }} +{{- end }} +{{- if .Values.dashboards }} +{{- range $provider, $dashboards := .Values.dashboards }} +{{- range $key, $value := $dashboards }} +{{- if (or (hasKey $value "json") (hasKey $value "file")) }} + - name: dashboards-{{ $provider }} + mountPath: "/var/lib/grafana/dashboards/{{ $provider }}/{{ $key }}.json" + subPath: "{{ $key }}.json" +{{- end }} +{{- end }} +{{- end }} +{{- end -}} +{{- if .Values.dashboardsConfigMaps }} +{{- range (keys .Values.dashboardsConfigMaps | sortAlpha) }} + - name: dashboards-{{ . }} + mountPath: "/var/lib/grafana/dashboards/{{ . }}" +{{- end }} +{{- end }} +{{- if .Values.datasources }} + - name: config + mountPath: "/etc/grafana/provisioning/datasources/datasources.yaml" + subPath: datasources.yaml +{{- end }} +{{- if .Values.notifiers }} + - name: config + mountPath: "/etc/grafana/provisioning/notifiers/notifiers.yaml" + subPath: notifiers.yaml +{{- end }} +{{- if .Values.dashboardProviders }} + - name: config + mountPath: "/etc/grafana/provisioning/dashboards/dashboardproviders.yaml" + subPath: dashboardproviders.yaml +{{- end }} +{{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + mountPath: {{ .Values.sidecar.dashboards.folder | quote }} +{{ if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + mountPath: "/etc/grafana/provisioning/dashboards/sc-dashboardproviders.yaml" + subPath: provider.yaml +{{- end}} +{{- end}} +{{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + mountPath: "/etc/grafana/provisioning/datasources" +{{- end}} +{{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + mountPath: "/etc/grafana/provisioning/notifiers" +{{- end}} + {{- range .Values.extraSecretMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + readOnly: {{ .readOnly }} + subPath: {{ .subPath | default "" }} + {{- end }} + {{- range .Values.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + {{- end }} + ports: + - name: {{ .Values.service.portName }} + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + - name: {{ .Values.podPortName }} + containerPort: 3000 + protocol: TCP + env: + {{- if not .Values.env.GF_SECURITY_ADMIN_USER }} + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ .Values.admin.existingSecret | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.userKey | default "admin-user" }} + {{- end }} + {{- if and (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) }} + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.admin.existingSecret | default (include "grafana.fullname" .) }} + key: {{ .Values.admin.passwordKey | default "admin-password" }} + {{- end }} + {{- if .Values.plugins }} + - name: GF_INSTALL_PLUGINS + valueFrom: + configMapKeyRef: + name: {{ template "grafana.fullname" . }} + key: plugins + {{- end }} + {{- if .Values.smtp.existingSecret }} + - name: GF_SMTP_USER + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.userKey | default "user" }} + - name: GF_SMTP_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.smtp.existingSecret }} + key: {{ .Values.smtp.passwordKey | default "password" }} + {{- end }} + {{- range $key, $value := .Values.envValueFrom }} + - name: {{ $key | quote }} + valueFrom: +{{ toYaml $value | indent 10 }} + {{- end }} +{{- range $key, $value := .Values.env }} + - name: "{{ $key }}" + value: "{{ $value }}" +{{- end }} + {{- if .Values.envFromSecret }} + envFrom: + - secretRef: + name: {{ tpl .Values.envFromSecret . }} + {{- end }} + {{- if .Values.envRenderSecret }} + envFrom: + - secretRef: + name: {{ template "grafana.fullname" . }}-env + {{- end }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 6 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 6 }} + resources: +{{ toYaml .Values.resources | indent 6 }} +{{- with .Values.extraContainers }} +{{ tpl . $ | indent 2 }} +{{- end }} +nodeSelector: {{ include "linux-node-selector" . | nindent 2 }} +{{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 2 }} +{{- end }} +{{- with .Values.affinity }} +affinity: +{{ toYaml . | indent 2 }} +{{- end }} +tolerations: {{ include "linux-node-tolerations" . | nindent 2 }} +{{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 2 }} +{{- end }} +volumes: + - name: config + configMap: + name: {{ template "grafana.fullname" . }} +{{- range .Values.extraConfigmapMounts }} + - name: {{ .name }} + configMap: + name: {{ .configMap }} +{{- end }} + {{- if .Values.dashboards }} + {{- range (keys .Values.dashboards | sortAlpha) }} + - name: dashboards-{{ . }} + configMap: + name: {{ template "grafana.fullname" $ }}-dashboards-{{ . }} + {{- end }} + {{- end }} + {{- if .Values.dashboardsConfigMaps }} + {{ $root := . }} + {{- range $provider, $name := .Values.dashboardsConfigMaps }} + - name: dashboards-{{ $provider }} + configMap: + name: {{ tpl $name $root }} + {{- end }} + {{- end }} + {{- if .Values.ldap.enabled }} + - name: ldap + secret: + {{- if .Values.ldap.existingSecret }} + secretName: {{ .Values.ldap.existingSecret }} + {{- else }} + secretName: {{ template "grafana.fullname" . }} + {{- end }} + items: + - key: ldap-toml + path: ldap.toml + {{- end }} +{{- if and .Values.persistence.enabled (eq .Values.persistence.type "pvc") }} + - name: storage + persistentVolumeClaim: + claimName: {{ .Values.persistence.existingClaim | default (include "grafana.fullname" .) }} +{{- else if and .Values.persistence.enabled (eq .Values.persistence.type "statefulset") }} +# nothing +{{- else }} + - name: storage + emptyDir: {} +{{- end -}} +{{- if .Values.sidecar.dashboards.enabled }} + - name: sc-dashboard-volume + emptyDir: {} +{{- if .Values.sidecar.dashboards.SCProvider }} + - name: sc-dashboard-provider + configMap: + name: {{ template "grafana.fullname" . }}-config-dashboards +{{- end }} +{{- end }} +{{- if .Values.sidecar.datasources.enabled }} + - name: sc-datasources-volume + emptyDir: {} +{{- end -}} +{{- if .Values.sidecar.notifiers.enabled }} + - name: sc-notifiers-volume + emptyDir: {} +{{- end -}} +{{- range .Values.extraSecretMounts }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + defaultMode: {{ .defaultMode }} +{{- end }} +{{- range .Values.extraVolumeMounts }} + - name: {{ .name }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} +{{- end }} +{{- range .Values.extraEmptyDirMounts }} + - name: {{ .name }} + emptyDir: {} +{{- end -}} +{{- if .Values.extraContainerVolumes }} +{{ toYaml .Values.extraContainerVolumes | indent 2 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/clusterrole.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/clusterrole.yaml new file mode 100755 index 000000000..b3ef6ab3b --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/clusterrole.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.rbac.create (not .Values.rbac.namespaced) }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} + name: {{ template "grafana.fullname" . }}-clusterrole +{{- if or .Values.sidecar.dashboards.enabled (or .Values.sidecar.datasources.enabled .Values.rbac.extraClusterRoleRules) }} +rules: +{{- if or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled }} +- apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] +{{- end}} +{{- with .Values.rbac.extraClusterRoleRules }} +{{ toYaml . | indent 0 }} +{{- end}} +{{- else }} +rules: [] +{{- end}} +{{- end}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..8ee08b2aa --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create (not .Values.rbac.namespaced) }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "grafana.fullname" . }}-clusterrolebinding + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +subjects: + - kind: ServiceAccount + name: {{ template "grafana.serviceAccountName" . }} + namespace: {{ template "grafana.namespace" . }} +roleRef: + kind: ClusterRole + name: {{ template "grafana.fullname" . }}-clusterrole + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/configmap-dashboard-provider.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/configmap-dashboard-provider.yaml new file mode 100755 index 000000000..af5d464b6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/configmap-dashboard-provider.yaml @@ -0,0 +1,25 @@ +{{- if .Values.sidecar.dashboards.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} + name: {{ template "grafana.fullname" . }}-config-dashboards + namespace: {{ template "grafana.namespace" . }} +data: + provider.yaml: |- + apiVersion: 1 + providers: + - name: '{{ .Values.sidecar.dashboards.provider.name }}' + orgId: {{ .Values.sidecar.dashboards.provider.orgid }} + folder: '{{ .Values.sidecar.dashboards.provider.folder }}' + type: {{ .Values.sidecar.dashboards.provider.type }} + disableDeletion: {{ .Values.sidecar.dashboards.provider.disableDelete }} + allowUiUpdates: {{ .Values.sidecar.dashboards.provider.allowUiUpdates }} + options: + path: {{ .Values.sidecar.dashboards.folder }}{{- with .Values.sidecar.dashboards.defaultFolderName }}/{{ . }}{{- end }} +{{- end}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/configmap.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/configmap.yaml new file mode 100755 index 000000000..a9fdc3a8f --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/configmap.yaml @@ -0,0 +1,69 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: +{{- if .Values.plugins }} + plugins: {{ join "," .Values.plugins }} +{{- end }} + grafana.ini: | +{{- range $key, $value := index .Values "grafana.ini" }} + [{{ $key }}] + {{- range $elem, $elemVal := $value }} + {{ $elem }} = {{ $elemVal }} + {{- end }} +{{- end }} + +{{- if .Values.datasources }} +{{ $root := . }} + {{- range $key, $value := .Values.datasources }} + {{ $key }}: | +{{ tpl (toYaml $value | indent 4) $root }} + {{- end -}} +{{- end -}} + +{{- if .Values.notifiers }} + {{- range $key, $value := .Values.notifiers }} + {{ $key }}: | +{{ toYaml $value | indent 4 }} + {{- end -}} +{{- end -}} + +{{- if .Values.dashboardProviders }} + {{- range $key, $value := .Values.dashboardProviders }} + {{ $key }}: | +{{ toYaml $value | indent 4 }} + {{- end -}} +{{- end -}} + +{{- if .Values.dashboards }} + download_dashboards.sh: | + #!/usr/bin/env sh + set -euf + {{- if .Values.dashboardProviders }} + {{- range $key, $value := .Values.dashboardProviders }} + {{- range $value.providers }} + mkdir -p {{ .options.path }} + {{- end }} + {{- end }} + {{- end }} + + {{- range $provider, $dashboards := .Values.dashboards }} + {{- range $key, $value := $dashboards }} + {{- if (or (hasKey $value "gnetId") (hasKey $value "url")) }} + curl -skf \ + --connect-timeout 60 \ + --max-time 60 \ + {{- if not $value.b64content }} + -H "Accept: application/json" \ + -H "Content-Type: application/json;charset=UTF-8" \ + {{ end }} + {{- if $value.url -}}"{{ $value.url }}"{{- else -}}"https://grafana.com/api/dashboards/{{ $value.gnetId }}/revisions/{{- if $value.revision -}}{{ $value.revision }}{{- else -}}1{{- end -}}/download"{{- end -}}{{ if $value.datasource }} | sed '/-- .* --/! s/"datasource":.*,/"datasource": "{{ $value.datasource }}",/g'{{ end }}{{- if $value.b64content -}} | base64 -d {{- end -}} \ + > "/var/lib/grafana/dashboards/{{ $provider }}/{{ $key }}.json" + {{- end -}} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/dashboards-json-configmap.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/dashboards-json-configmap.yaml new file mode 100755 index 000000000..59e0be641 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/dashboards-json-configmap.yaml @@ -0,0 +1,35 @@ +{{- if .Values.dashboards }} +{{ $files := .Files }} +{{- range $provider, $dashboards := .Values.dashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "grafana.fullname" $ }}-dashboards-{{ $provider }} + namespace: {{ template "grafana.namespace" $ }} + labels: + {{- include "grafana.labels" $ | nindent 4 }} + dashboard-provider: {{ $provider }} +{{- if $dashboards }} +data: +{{- $dashboardFound := false }} +{{- range $key, $value := $dashboards }} +{{- if (or (hasKey $value "json") (hasKey $value "file")) }} +{{- $dashboardFound = true }} +{{ print $key | indent 2 }}.json: +{{- if hasKey $value "json" }} + |- +{{ $value.json | indent 6 }} +{{- end }} +{{- if hasKey $value "file" }} +{{ toYaml ( $files.Get $value.file ) | indent 4}} +{{- end }} +{{- end }} +{{- end }} +{{- if not $dashboardFound }} + {} +{{- end }} +{{- end }} +--- +{{- end }} + +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/deployment.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/deployment.yaml new file mode 100755 index 000000000..9d134bde3 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/deployment.yaml @@ -0,0 +1,47 @@ +{{ if (or (not .Values.persistence.enabled) (eq .Values.persistence.type "pvc")) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.labels }} +{{ toYaml .Values.labels | indent 4 }} +{{- end }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} +{{- with .Values.deploymentStrategy }} + strategy: +{{ toYaml . | trim | indent 4 }} +{{- end }} + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} +{{- with .Values.podLabels }} +{{ toYaml . | indent 8 }} +{{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + checksum/sc-dashboard-provider-config: {{ include (print $.Template.BasePath "/configmap-dashboard-provider.yaml") . | sha256sum }} +{{- if and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} +{{- end }} +{{- if .Values.envRenderSecret }} + checksum/secret-env: {{ include (print $.Template.BasePath "/secret-env.yaml") . | sha256sum }} +{{- end }} +{{- with .Values.podAnnotations }} +{{ toYaml . | indent 8 }} +{{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/headless-service.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/headless-service.yaml new file mode 100755 index 000000000..2fa816e04 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/headless-service.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "statefulset")}} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "grafana.fullname" . }}-headless + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + clusterIP: None + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} + type: ClusterIP +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/ingress.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/ingress.yaml new file mode 100755 index 000000000..8d35662e7 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/ingress.yaml @@ -0,0 +1,55 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "grafana.fullname" . -}} +{{- $servicePort := .Values.service.port -}} +{{- $ingressPath := .Values.ingress.path -}} +{{- $extraPaths := .Values.ingress.extraPaths -}} +{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} +apiVersion: networking.k8s.io/v1beta1 +{{ else }} +apiVersion: extensions/v1beta1 +{{ end -}} +kind: Ingress +metadata: + name: {{ $fullName }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.ingress.labels }} +{{ toYaml .Values.ingress.labels | indent 4 }} +{{- end }} + {{- if .Values.ingress.annotations }} + annotations: + {{- range $key, $value := .Values.ingress.annotations }} + {{ $key }}: {{ tpl $value $ | quote }} + {{- end }} + {{- end }} +spec: +{{- if .Values.ingress.tls }} + tls: +{{ toYaml .Values.ingress.tls | indent 4 }} +{{- end }} + rules: + {{- if .Values.ingress.hosts }} + {{- range .Values.ingress.hosts }} + - host: {{ . }} + http: + paths: +{{ if $extraPaths }} +{{ toYaml $extraPaths | indent 10 }} +{{- end }} + - path: {{ $ingressPath }} + backend: + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- else }} + - http: + paths: + - backend: + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- if $ingressPath }} + path: {{ $ingressPath }} + {{- end }} + {{- end -}} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/nginx-config.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/nginx-config.yaml new file mode 100755 index 000000000..f847c51ce --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/nginx-config.yaml @@ -0,0 +1,75 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-nginx-proxy-config + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + server { + listen 8080; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location /api/dashboards { + proxy_pass http://localhost:3000; + } + + location /api/search { + proxy_pass http://localhost:3000; + + sub_filter_types application/json; + sub_filter_once off; + sub_filter '"url":"/d' '"url":"d'; + } + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:3000/; + + sub_filter_types text/html; + sub_filter_once off; + sub_filter '"appSubUrl":""' '"appSubUrl":"."'; + sub_filter '"url":"/' '"url":"./'; + sub_filter ':"/avatar/' ':"avatar/'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + } + } + } diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/poddisruptionbudget.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/poddisruptionbudget.yaml new file mode 100755 index 000000000..61813a436 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/poddisruptionbudget.yaml @@ -0,0 +1,22 @@ +{{- if .Values.podDisruptionBudget }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.labels }} +{{ toYaml .Values.labels | indent 4 }} +{{- end }} +spec: +{{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} +{{- end }} +{{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} +{{- end }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/podsecuritypolicy.yaml new file mode 100755 index 000000000..ba835efeb --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/podsecuritypolicy.yaml @@ -0,0 +1,48 @@ +{{- if .Values.rbac.pspEnabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.rbac.pspAnnotations }} + annotations: {{ toYaml .Values.rbac.pspAnnotations | nindent 4 }} +{{- end }} +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + # Default set from Docker, without DAC_OVERRIDE or CHOWN + - FOWNER + - FSETID + - KILL + - SETGID + - SETUID + - SETPCAP + - NET_BIND_SERVICE + - NET_RAW + - SYS_CHROOT + - MKNOD + - AUDIT_WRITE + - SETFCAP + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/pvc.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/pvc.yaml new file mode 100755 index 000000000..4727d0aa1 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/pvc.yaml @@ -0,0 +1,28 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "pvc")}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- with .Values.persistence.annotations }} + annotations: +{{ toYaml . | indent 4 }} + {{- end }} + {{- with .Values.persistence.finalizers }} + finalizers: +{{ toYaml . | indent 4 }} + {{- end }} +spec: + accessModes: + {{- range .Values.persistence.accessModes }} + - {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- if .Values.persistence.storageClassName }} + storageClassName: {{ .Values.persistence.storageClassName }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/role.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/role.yaml new file mode 100755 index 000000000..c95c1d042 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/role.yaml @@ -0,0 +1,32 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +{{- if or .Values.rbac.pspEnabled (and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled (or .Values.sidecar.datasources.enabled .Values.rbac.extraRoleRules))) }} +rules: +{{- if .Values.rbac.pspEnabled }} +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ template "grafana.fullname" . }}] +{{- end }} +{{- if and .Values.rbac.namespaced (or .Values.sidecar.dashboards.enabled .Values.sidecar.datasources.enabled) }} +- apiGroups: [""] # "" indicates the core API group + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] +{{- end }} +{{- with .Values.rbac.extraRoleRules }} +{{ toYaml . | indent 0 }} +{{- end}} +{{- else }} +rules: [] +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/rolebinding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/rolebinding.yaml new file mode 100755 index 000000000..beaf2f003 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "grafana.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "grafana.serviceAccountName" . }} + namespace: {{ template "grafana.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/secret-env.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/secret-env.yaml new file mode 100755 index 000000000..5c09313e6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/secret-env.yaml @@ -0,0 +1,14 @@ +{{- if .Values.envRenderSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "grafana.fullname" . }}-env + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +type: Opaque +data: +{{- range $key, $val := .Values.envRenderSecret }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end -}} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/secret.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/secret.yaml new file mode 100755 index 000000000..9d2f072e0 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/secret.yaml @@ -0,0 +1,22 @@ +{{- if or (and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD)) (and .Values.ldap.enabled (not .Values.ldap.existingSecret)) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +type: Opaque +data: + {{- if and (not .Values.admin.existingSecret) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD__FILE) (not .Values.env.GF_SECURITY_ADMIN_PASSWORD) }} + admin-user: {{ .Values.adminUser | b64enc | quote }} + {{- if .Values.adminPassword }} + admin-password: {{ .Values.adminPassword | b64enc | quote }} + {{- else }} + admin-password: {{ randAlphaNum 40 | b64enc | quote }} + {{- end }} + {{- end }} + {{- if not .Values.ldap.existingSecret }} + ldap-toml: {{ .Values.ldap.config | b64enc | quote }} + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/service.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/service.yaml new file mode 100755 index 000000000..276456698 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/service.yaml @@ -0,0 +1,50 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- if .Values.service.labels }} +{{ toYaml .Values.service.labels | indent 4 }} +{{- end }} +{{- with .Values.service.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: +{{- if (or (eq .Values.service.type "ClusterIP") (empty .Values.service.type)) }} + type: ClusterIP + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{end}} +{{- else if eq .Values.service.type "LoadBalancer" }} + type: {{ .Values.service.type }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + {{- if .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: +{{ toYaml .Values.service.loadBalancerSourceRanges | indent 4 }} + {{- end -}} +{{- else }} + type: {{ .Values.service.type }} +{{- end }} +{{- if .Values.service.externalIPs }} + externalIPs: +{{ toYaml .Values.service.externalIPs | indent 4 }} +{{- end }} + ports: + - name: {{ .Values.service.portName }} + port: {{ .Values.service.port }} + protocol: TCP + targetPort: {{ .Values.service.targetPort }} +{{ if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }} + nodePort: {{.Values.service.nodePort}} +{{ end }} + {{- if .Values.extraExposePorts }} + {{- tpl (toYaml .Values.extraExposePorts) . | indent 4 }} + {{- end }} + selector: + {{- include "grafana.selectorLabels" . | nindent 4 }} + diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/serviceaccount.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/serviceaccount.yaml new file mode 100755 index 000000000..7576eeef0 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.serviceAccount.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} + name: {{ template "grafana.serviceAccountName" . }} + namespace: {{ template "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/servicemonitor.yaml new file mode 100755 index 000000000..988956bcd --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/servicemonitor.yaml @@ -0,0 +1,36 @@ +{{- if .Values.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "grafana.fullname" . }} + {{- if .Values.serviceMonitor.namespace }} + namespace: {{ .Values.serviceMonitor.namespace }} + {{- end }} + labels: + {{- include "grafana.labels" . | nindent 4 }} + {{- if .Values.serviceMonitor.labels }} + {{- toYaml .Values.serviceMonitor.labels | nindent 4 }} + {{- end }} +spec: + endpoints: + - interval: {{ .Values.serviceMonitor.interval }} + {{- if .Values.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: true + port: {{ .Values.service.portName }} + path: {{ .Values.serviceMonitor.path }} + {{- if .Values.serviceMonitor.relabelings }} + relabelings: + {{- toYaml .Values.serviceMonitor.relabelings | nindent 4 }} + {{- end }} + jobLabel: "{{ .Release.Name }}" + selector: + matchLabels: + app: {{ template "grafana.name" . }} + release: "{{ .Release.Name }}" + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/statefulset.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/statefulset.yaml new file mode 100755 index 000000000..732dd8e53 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/statefulset.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) (eq .Values.persistence.type "statefulset")}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "grafana.fullname" . }} + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +{{- with .Values.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + {{- include "grafana.selectorLabels" . | nindent 6 }} + serviceName: {{ template "grafana.fullname" . }}-headless + template: + metadata: + labels: + {{- include "grafana.selectorLabels" . | nindent 8 }} +{{- with .Values.podLabels }} +{{ toYaml . | indent 8 }} +{{- end }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + checksum/dashboards-json-config: {{ include (print $.Template.BasePath "/dashboards-json-configmap.yaml") . | sha256sum }} + checksum/sc-dashboard-provider-config: {{ include (print $.Template.BasePath "/configmap-dashboard-provider.yaml") . | sha256sum }} +{{- if not .Values.admin.existingSecret }} + checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} +{{- end }} +{{- with .Values.podAnnotations }} +{{ toYaml . | indent 8 }} +{{- end }} + spec: + {{- include "grafana.pod" . | nindent 6 }} + volumeClaimTemplates: + - metadata: + name: storage + spec: + accessModes: {{ .Values.persistence.accessModes }} + storageClassName: {{ .Values.persistence.storageClassName }} + resources: + requests: + storage: {{ .Values.persistence.size }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-configmap.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-configmap.yaml new file mode 100755 index 000000000..ff53aaf1b --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-configmap.yaml @@ -0,0 +1,17 @@ +{{- if .Values.testFramework.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "grafana.fullname" . }}-test + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +data: + run.sh: |- + @test "Test Health" { + url="http://{{ template "grafana.fullname" . }}/api/health" + + code=$(wget --server-response --spider --timeout 10 --tries 1 ${url} 2>&1 | awk '/^ HTTP/{print $2}') + [ "$code" == "200" ] + } +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-podsecuritypolicy.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-podsecuritypolicy.yaml new file mode 100755 index 000000000..eb5cbbcd7 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-podsecuritypolicy.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.testFramework.enabled .Values.rbac.pspEnabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "grafana.fullname" . }}-test + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +spec: + allowPrivilegeEscalation: true + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + fsGroup: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + runAsUser: + rule: RunAsAny + volumes: + - configMap + - downwardAPI + - emptyDir + - projected + - secret +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-role.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-role.yaml new file mode 100755 index 000000000..6b10677ae --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-role.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.testFramework.enabled .Values.rbac.pspEnabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "grafana.fullname" . }}-test + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ template "grafana.fullname" . }}-test] +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-rolebinding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-rolebinding.yaml new file mode 100755 index 000000000..58fa5e78b --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-rolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.testFramework.enabled .Values.rbac.pspEnabled -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "grafana.fullname" . }}-test + namespace: {{ template "grafana.namespace" . }} + labels: + {{- include "grafana.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "grafana.fullname" . }}-test +subjects: +- kind: ServiceAccount + name: {{ template "grafana.serviceAccountNameTest" . }} + namespace: {{ template "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-serviceaccount.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-serviceaccount.yaml new file mode 100755 index 000000000..5c3350733 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test-serviceaccount.yaml @@ -0,0 +1,9 @@ +{{- if and .Values.testFramework.enabled .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + {{- include "grafana.labels" . | nindent 4 }} + name: {{ template "grafana.serviceAccountNameTest" . }} + namespace: {{ template "grafana.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test.yaml new file mode 100755 index 000000000..cdc86e5f2 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/templates/tests/test.yaml @@ -0,0 +1,48 @@ +{{- if .Values.testFramework.enabled }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ template "grafana.fullname" . }}-test + labels: + {{- include "grafana.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test-success + namespace: {{ template "grafana.namespace" . }} +spec: + serviceAccountName: {{ template "grafana.serviceAccountNameTest" . }} + {{- if .Values.testFramework.securityContext }} + securityContext: {{ toYaml .Values.testFramework.securityContext | nindent 4 }} + {{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end}} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 4 }} + {{- end }} + {{- with .Values.affinity }} + affinity: +{{ toYaml . | indent 4 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: +{{ toYaml . | indent 4 }} + {{- end }} + containers: + - name: {{ .Release.Name }}-test + image: "{{ .Values.testFramework.image}}:{{ .Values.testFramework.tag }}" + imagePullPolicy: "{{ .Values.testFramework.imagePullPolicy}}" + command: ["/opt/bats/bin/bats", "-t", "/tests/run.sh"] + volumeMounts: + - mountPath: /tests + name: tests + readOnly: true + volumes: + - name: tests + configMap: + name: {{ template "grafana.fullname" . }}-test + restartPolicy: Never +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/values.yaml new file mode 100755 index 000000000..e4ce42482 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/grafana/values.yaml @@ -0,0 +1,562 @@ +rbac: + create: true + pspEnabled: true + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + namespaced: false + extraRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] + extraClusterRoleRules: [] + # - apiGroups: [] + # resources: [] + # verbs: [] +serviceAccount: + create: true + name: + nameTest: +# annotations: + +replicas: 1 + +## See `kubectl explain poddisruptionbudget.spec` for more +## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} +# minAvailable: 1 +# maxUnavailable: 1 + +## See `kubectl explain deployment.spec.strategy` for more +## ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy +deploymentStrategy: + type: RollingUpdate + +readinessProbe: + httpGet: + path: /api/health + port: 3000 + +livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + failureThreshold: 10 + +## Use an alternate scheduler, e.g. "stork". +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: "default-scheduler" + +image: + repository: rancher/mirrored-grafana-grafana + tag: 7.1.5 + sha: "" + pullPolicy: IfNotPresent + + ## Optionally specify an array of imagePullSecrets. + ## Secrets must be manually created in the namespace. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + # pullSecrets: + # - myRegistrKeySecretName + +testFramework: + enabled: true + image: "rancher/mirrored-bats-bats" + tag: "v1.1.0" + imagePullPolicy: IfNotPresent + securityContext: + runAsNonRoot: true + runAsUser: 1000 + +securityContext: + runAsNonRoot: true + runAsUser: 472 + runAsGroup: 472 + fsGroup: 472 + + +extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # subPath: certificates.crt # (optional) + # configMap: certs-configmap + # readOnly: true + + +extraEmptyDirMounts: [] + # - name: provisioning-notifiers + # mountPath: /etc/grafana/provisioning/notifiers + + +## Assign a PriorityClassName to pods if set +# priorityClassName: + +downloadDashboardsImage: + repository: rancher/mirrored-curlimages-curl + tag: 7.70.0 + sha: "" + pullPolicy: IfNotPresent + +downloadDashboards: + env: {} + resources: {} + +## Pod Annotations +# podAnnotations: {} + +## Pod Labels +# podLabels: {} + +podPortName: grafana + +## Deployment annotations +# annotations: {} + +## Expose the grafana service to be accessed from outside the cluster (LoadBalancer service). +## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it. +## ref: http://kubernetes.io/docs/user-guide/services/ +## +service: + type: ClusterIP + port: 80 + targetPort: 3000 + # targetPort: 4181 To be used with a proxy extraContainer + annotations: {} + labels: {} + portName: service + +serviceMonitor: + ## If true, a ServiceMonitor CRD is created for a prometheus operator + ## https://github.com/coreos/prometheus-operator + ## + enabled: false + path: /metrics + # namespace: monitoring (defaults to use the namespace this chart is deployed to) + labels: {} + interval: 1m + scrapeTimeout: 30s + relabelings: [] + +extraExposePorts: [] + # - name: keycloak + # port: 8080 + # targetPort: 8080 + # type: ClusterIP + +# overrides pod.spec.hostAliases in the grafana deployment's pods +hostAliases: [] + # - ip: "1.2.3.4" + # hostnames: + # - "my.host.com" + +ingress: + enabled: false + # Values can be templated + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + labels: {} + path: / + hosts: + - chart-example.local + ## Extra paths to prepend to every host configuration. This is useful when working with annotation based services. + extraPaths: [] + # - path: /* + # backend: + # serviceName: ssl-redirect + # servicePort: use-annotation + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} +# limits: +# cpu: 100m +# memory: 128Mi +# requests: +# cpu: 100m +# memory: 128Mi + +## Node labels for pod assignment +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +# +nodeSelector: {} + +## Tolerations for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Affinity for pod assignment +## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +extraInitContainers: [] + +## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod +extraContainers: | +# - name: proxy +# image: quay.io/gambol99/keycloak-proxy:latest +# args: +# - -provider=github +# - -client-id= +# - -client-secret= +# - -github-org= +# - -email-domain=* +# - -cookie-secret= +# - -http-address=http://0.0.0.0:4181 +# - -upstream-url=http://127.0.0.1:3000 +# ports: +# - name: proxy-web +# containerPort: 4181 + +## Volumes that can be used in init containers that will not be mounted to deployment pods +extraContainerVolumes: [] +# - name: volume-from-secret +# secret: +# secretName: secret-to-mount +# - name: empty-dir-volume +# emptyDir: {} + +## Enable persistence using Persistent Volume Claims +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + type: pvc + enabled: false + # storageClassName: default + accessModes: + - ReadWriteOnce + size: 10Gi + # annotations: {} + finalizers: + - kubernetes.io/pvc-protection + # subPath: "" + # existingClaim: + +initChownData: + ## If false, data ownership will not be reset at startup + ## This allows the prometheus-server to be run with an arbitrary user + ## + enabled: true + + ## initChownData container image + ## + image: + repository: rancher/mirrored-library-busybox + tag: "1.31.1" + sha: "" + pullPolicy: IfNotPresent + + ## initChownData resource requests and limits + ## Ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + +# Administrator credentials when not using an existing secret (see below) +adminUser: admin +# adminPassword: strongpassword + +# Use an existing secret for the admin user. +admin: + existingSecret: "" + userKey: admin-user + passwordKey: admin-password + +## Define command to be executed at startup by grafana container +## Needed if using `vault-env` to manage secrets (ref: https://banzaicloud.com/blog/inject-secrets-into-pods-vault/) +## Default is "run.sh" as defined in grafana's Dockerfile +# command: +# - "sh" +# - "/run.sh" + +## Use an alternate scheduler, e.g. "stork". +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +# schedulerName: + +## Extra environment variables that will be pass onto deployment pods +env: {} + +## "valueFrom" environment variable references that will be added to deployment pods +## ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#envvarsource-v1-core +## Renders in container spec as: +## env: +## ... +## - name: +## valueFrom: +## +envValueFrom: {} + +## The name of a secret in the same kubernetes namespace which contain values to be added to the environment +## This can be useful for auth tokens, etc. Value is templated. +envFromSecret: "" + +## Sensible environment variables that will be rendered as new secret object +## This can be useful for auth tokens, etc +envRenderSecret: {} + +## Additional grafana server secret mounts +# Defines additional mounts with secrets. Secrets must be manually created in the namespace. +extraSecretMounts: [] + # - name: secret-files + # mountPath: /etc/secrets + # secretName: grafana-secret-files + # readOnly: true + # subPath: "" + +## Additional grafana server volume mounts +# Defines additional volume mounts. +extraVolumeMounts: [] + # - name: extra-volume + # mountPath: /mnt/volume + # readOnly: true + # existingClaim: volume-claim + +## Pass the plugins you want installed as a list. +## +plugins: [] + # - digrich-bubblechart-panel + # - grafana-clock-panel + +## Configure grafana datasources +## ref: http://docs.grafana.org/administration/provisioning/#datasources +## +datasources: {} +# datasources.yaml: +# apiVersion: 1 +# datasources: +# - name: Prometheus +# type: prometheus +# url: http://prometheus-prometheus-server +# access: proxy +# isDefault: true + +## Configure notifiers +## ref: http://docs.grafana.org/administration/provisioning/#alert-notification-channels +## +notifiers: {} +# notifiers.yaml: +# notifiers: +# - name: email-notifier +# type: email +# uid: email1 +# # either: +# org_id: 1 +# # or +# org_name: Main Org. +# is_default: true +# settings: +# addresses: an_email_address@example.com +# delete_notifiers: + +## Configure grafana dashboard providers +## ref: http://docs.grafana.org/administration/provisioning/#dashboards +## +## `path` must be /var/lib/grafana/dashboards/ +## +dashboardProviders: {} +# dashboardproviders.yaml: +# apiVersion: 1 +# providers: +# - name: 'default' +# orgId: 1 +# folder: '' +# type: file +# disableDeletion: false +# editable: true +# options: +# path: /var/lib/grafana/dashboards/default + +## Configure grafana dashboard to import +## NOTE: To use dashboards you must also enable/configure dashboardProviders +## ref: https://grafana.com/dashboards +## +## dashboards per provider, use provider name as key. +## +dashboards: {} + # default: + # some-dashboard: + # json: | + # $RAW_JSON + # custom-dashboard: + # file: dashboards/custom-dashboard.json + # prometheus-stats: + # gnetId: 2 + # revision: 2 + # datasource: Prometheus + # local-dashboard: + # url: https://example.com/repository/test.json + # local-dashboard-base64: + # url: https://example.com/repository/test-b64.json + # b64content: true + +## Reference to external ConfigMap per provider. Use provider name as key and ConfigMap name as value. +## A provider dashboards must be defined either by external ConfigMaps or in values.yaml, not in both. +## ConfigMap data example: +## +## data: +## example-dashboard.json: | +## RAW_JSON +## +dashboardsConfigMaps: {} +# default: "" + +## Grafana's primary configuration +## NOTE: values in map will be converted to ini format +## ref: http://docs.grafana.org/installation/configuration/ +## +grafana.ini: + paths: + data: /var/lib/grafana/data + logs: /var/log/grafana + plugins: /var/lib/grafana/plugins + provisioning: /etc/grafana/provisioning + analytics: + check_for_updates: true + log: + mode: console + grafana_net: + url: https://grafana.net +## grafana Authentication can be enabled with the following values on grafana.ini + # server: + # The full public facing url you use in browser, used for redirects and emails + # root_url: + # https://grafana.com/docs/grafana/latest/auth/github/#enable-github-in-grafana + # auth.github: + # enabled: false + # allow_sign_up: false + # scopes: user:email,read:org + # auth_url: https://github.com/login/oauth/authorize + # token_url: https://github.com/login/oauth/access_token + # api_url: https://github.com/user + # team_ids: + # allowed_organizations: + # client_id: + # client_secret: +## LDAP Authentication can be enabled with the following values on grafana.ini +## NOTE: Grafana will fail to start if the value for ldap.toml is invalid + # auth.ldap: + # enabled: true + # allow_sign_up: true + # config_file: /etc/grafana/ldap.toml + +## Grafana's LDAP configuration +## Templated by the template in _helpers.tpl +## NOTE: To enable the grafana.ini must be configured with auth.ldap.enabled +## ref: http://docs.grafana.org/installation/configuration/#auth-ldap +## ref: http://docs.grafana.org/installation/ldap/#configuration +ldap: + enabled: false + # `existingSecret` is a reference to an existing secret containing the ldap configuration + # for Grafana in a key `ldap-toml`. + existingSecret: "" + # `config` is the content of `ldap.toml` that will be stored in the created secret + config: "" + # config: |- + # verbose_logging = true + + # [[servers]] + # host = "my-ldap-server" + # port = 636 + # use_ssl = true + # start_tls = false + # ssl_skip_verify = false + # bind_dn = "uid=%s,ou=users,dc=myorg,dc=com" + +## Grafana's SMTP configuration +## NOTE: To enable, grafana.ini must be configured with smtp.enabled +## ref: http://docs.grafana.org/installation/configuration/#smtp +smtp: + # `existingSecret` is a reference to an existing secret containing the smtp configuration + # for Grafana. + existingSecret: "" + userKey: "user" + passwordKey: "password" + +## Sidecars that collect the configmaps with specified label and stores the included files them into the respective folders +## Requires at least Grafana 5 to work and can't be used together with parameters dashboardProviders, datasources and dashboards +sidecar: + image: + repository: rancher/mirrored-kiwigrid-k8s-sidecar + tag: 0.1.151 + sha: "" + imagePullPolicy: IfNotPresent + resources: {} +# limits: +# cpu: 100m +# memory: 100Mi +# requests: +# cpu: 50m +# memory: 50Mi + # skipTlsVerify Set to true to skip tls verification for kube api calls + # skipTlsVerify: true + enableUniqueFilenames: false + dashboards: + enabled: false + SCProvider: true + # label that the configmaps with dashboards are marked with + label: grafana_dashboard + # folder in the pod that should hold the collected dashboards (unless `defaultFolderName` is set) + folder: /tmp/dashboards + # The default folder name, it will create a subfolder under the `folder` and put dashboards in there instead + defaultFolderName: null + # If specified, the sidecar will search for dashboard config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + # provider configuration that lets grafana manage the dashboards + provider: + # name of the provider, should be unique + name: sidecarProvider + # orgid as configured in grafana + orgid: 1 + # folder in which the dashboards should be imported in grafana + folder: '' + # type of the provider + type: file + # disableDelete to activate a import-only behaviour + disableDelete: false + # allow updating provisioned dashboards from the UI + allowUiUpdates: false + datasources: + enabled: false + # label that the configmaps with datasources are marked with + label: grafana_datasource + # If specified, the sidecar will search for datasource config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + notifiers: + enabled: false + # label that the configmaps with notifiers are marked with + label: grafana_notifier + # If specified, the sidecar will search for notifier config-maps inside this namespace. + # Otherwise the namespace in which the sidecar is running will be used. + # It's also possible to specify ALL to search in all namespaces + searchNamespace: null + +## Override the deployment namespace +## +namespaceOverride: "" diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/.helmignore new file mode 100755 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/Chart.yaml new file mode 100755 index 000000000..56ff36fc7 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/Chart.yaml @@ -0,0 +1,13 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: k3sServer +type: application +version: 0.1.3 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/README.md new file mode 100755 index 000000000..dcecc69da --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/README.md @@ -0,0 +1,54 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/_helpers.tpl new file mode 100755 index 000000000..f77b8edf4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/_helpers.tpl @@ -0,0 +1,87 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) .Release.Namespace (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +release: {{ .Release.Name | quote }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-clients-rbac.yaml new file mode 100755 index 000000000..95346dee6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,74 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-clients.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-clients.yaml new file mode 100755 index 000000000..ed78792e5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-clients.yaml @@ -0,0 +1,135 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + - --metrics-addr=$(PORT) + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: PORT + value: :{{ .Values.clients.port }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-proxy-rbac.yaml new file mode 100755 index 000000000..a3509c160 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-proxy.yaml new file mode 100755 index 000000000..571e13138 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-proxy.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-servicemonitor.yaml new file mode 100755 index 000000000..2f3d7e54c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,39 @@ +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: + - port: metrics + proxyUrl: {{ template "pushProxy.proxyUrl" . }} + {{- if .Values.clients.https.enabled }} + params: + _scheme: [https] + {{- end }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ include "pushProxy.client.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/values.yaml new file mode 100755 index 000000000..e1bcf79a5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/k3sServer/values.yaml @@ -0,0 +1,86 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Note: if using a cloud provider, the nodes that contain the PushProxy client must allow the node(s) that contain(s) +# the PushProxy proxy to communicate with it on port {{ .Values.clients.port }}. If you have special restrictions, +# (i.e. client should only run on etcd nodes and only control plane should have access to the port on the etcd node), +# you will need to set the clients / proxy nodeSelector and tolerations accordingly + +# Configuration + +global: + cattle: + systemDefaultRegistry: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-client + tag: v0.1.0-rancher1-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.0-rancher1-proxy + command: ["pushprox-proxy"] \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/.helmignore new file mode 100755 index 000000000..f0c131944 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/Chart.yaml new file mode 100755 index 000000000..7114453ee --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/Chart.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +appVersion: 1.9.7 +description: Install kube-state-metrics to generate and expose cluster-level metrics +home: https://github.com/kubernetes/kube-state-metrics/ +keywords: +- metric +- monitoring +- prometheus +- kubernetes +maintainers: +- email: jose@armesto.net + name: fiunchinho +- email: tariq.ibrahim@mulesoft.com + name: tariq1890 +- email: manuel@rueg.eu + name: mrueg +name: kube-state-metrics +sources: +- https://github.com/kubernetes/kube-state-metrics/ +version: 2.8.14 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/README.md new file mode 100755 index 000000000..8a27b554d --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/README.md @@ -0,0 +1,80 @@ +# kube-state-metrics Helm Chart + +* Installs the [kube-state-metrics agent](https://github.com/kubernetes/kube-state-metrics). + +## Installing the Chart + +To install the chart with the release name `my-release`: + +```bash +$ helm install stable/kube-state-metrics +``` + +## Configuration + +| Parameter | Description | Default | +|:---------------------------------------------|:--------------------------------------------------------------------------------------|:-------------------------------------------| +| `image.repository` | The image repository to pull from | `quay.io/coreos/kube-state-metrics` | +| `image.tag` | The image tag to pull from | `v1.9.7` | +| `image.pullPolicy` | Image pull policy | `IfNotPresent` | +| `imagePullSecrets` | List of container registry secrets | `[]` | +| `replicas` | Number of replicas | `1` | +| `autosharding.enabled` | Set to `true` to automatically shard data across `replicas` pods. EXPERIMENTAL | `false` | +| `service.port` | The port of the container | `8080` | +| `service.annotations` | Annotations to be added to the service | `{}` | +| `customLabels` | Custom labels to apply to service, deployment and pods | `{}` | +| `hostNetwork` | Whether or not to use the host network | `false` | +| `prometheusScrape` | Whether or not enable prom scrape | `true` | +| `rbac.create` | If true, create & use RBAC resources | `true` | +| `serviceAccount.create` | If true, create & use serviceAccount | `true` | +| `serviceAccount.name` | If not set & create is true, use template fullname | | +| `serviceAccount.imagePullSecrets` | Specify image pull secrets field | `[]` | +| `serviceAccount.annotations` | Annotations to be added to the serviceAccount | `{}` | +| `podSecurityPolicy.enabled` | If true, create & use PodSecurityPolicy resources. Note that related RBACs are created only if `rbac.enabled` is `true`. | `false` | +| `podSecurityPolicy.annotations` | Specify pod annotations in the pod security policy | `{}` | +| `podSecurityPolicy.additionalVolumes` | Specify allowed volumes in the pod security policy (`secret` is always allowed) | `[]` | +| `securityContext.enabled` | Enable security context | `true` | +| `securityContext.fsGroup` | Group ID for the filesystem | `65534` | +| `securityContext.runAsGroup` | Group ID for the container | `65534` | +| `securityContext.runAsUser` | User ID for the container | `65534` | +| `priorityClassName` | Name of Priority Class to assign pods | `nil` | +| `nodeSelector` | Node labels for pod assignment | `{}` | +| `affinity` | Affinity settings for pod assignment | `{}` | +| `tolerations` | Tolerations for pod assignment | `[]` | +| `podAnnotations` | Annotations to be added to the pod | `{}` | +| `podDisruptionBudget` | Optional PodDisruptionBudget | `{}` | +| `resources` | kube-state-metrics resource requests and limits | `{}` | +| `collectors.certificatesigningrequests` | Enable the certificatesigningrequests collector. | `true` | +| `collectors.configmaps` | Enable the configmaps collector. | `true` | +| `collectors.cronjobs` | Enable the cronjobs collector. | `true` | +| `collectors.daemonsets` | Enable the daemonsets collector. | `true` | +| `collectors.deployments` | Enable the deployments collector. | `true` | +| `collectors.endpoints` | Enable the endpoints collector. | `true` | +| `collectors.horizontalpodautoscalers` | Enable the horizontalpodautoscalers collector. | `true` | +| `collectors.ingresses` | Enable the ingresses collector. | `true` | +| `collectors.jobs` | Enable the jobs collector. | `true` | +| `collectors.limitranges` | Enable the limitranges collector. | `true` | +| `collectors.mutatingwebhookconfigurations` | Enable the mutatingwebhookconfigurations collector. | `true` | +| `collectors.namespaces` | Enable the namespaces collector. | `true` | +| `collectors.networkpolicies` | Enable the networkpolicies collector. | `true` | +| `collectors.nodes` | Enable the nodes collector. | `true` | +| `collectors.persistentvolumeclaims` | Enable the persistentvolumeclaims collector. | `true` | +| `collectors.persistentvolumes` | Enable the persistentvolumes collector. | `true` | +| `collectors.poddisruptionbudgets` | Enable the poddisruptionbudgets collector. | `true` | +| `collectors.pods` | Enable the pods collector. | `true` | +| `collectors.replicasets` | Enable the replicasets collector. | `true` | +| `collectors.replicationcontrollers` | Enable the replicationcontrollers collector. | `true` | +| `collectors.resourcequotas` | Enable the resourcequotas collector. | `true` | +| `collectors.secrets` | Enable the secrets collector. | `true` | +| `collectors.services` | Enable the services collector. | `true` | +| `collectors.statefulsets` | Enable the statefulsets collector. | `true` | +| `collectors.storageclasses` | Enable the storageclasses collector. | `true` | +| `collectors.validatingwebhookconfigurations` | Enable the validatingwebhookconfigurations collector. | `true` | +| `collectors.verticalpodautoscalers` | Enable the verticalpodautoscalers collector. | `true` | +| `collectors.volumeattachments` | Enable the volumeattachments collector. | `true` | +| `prometheus.monitor.enabled` | Set this to `true` to create ServiceMonitor for Prometheus operator | `false` | +| `prometheus.monitor.additionalLabels` | Additional labels that can be used so ServiceMonitor will be discovered by Prometheus | `{}` | +| `prometheus.monitor.namespace` | Namespace where servicemonitor resource should be created | `the same namespace as kube-state-metrics` | +| `prometheus.monitor.honorLabels` | Honor metric labels | `false` | +| `namespaceOverride` | Override the deployment namespace | `""` (`Release.Namespace`) | +| `kubeTargetVersionOverride` | Override the k8s version of the target cluster | `""` | diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/NOTES.txt b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/NOTES.txt new file mode 100755 index 000000000..5a646e0cc --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/NOTES.txt @@ -0,0 +1,10 @@ +kube-state-metrics is a simple service that listens to the Kubernetes API server and generates metrics about the state of the objects. +The exposed metrics can be found here: +https://github.com/kubernetes/kube-state-metrics/blob/master/docs/README.md#exposed-metrics + +The metrics are exported on the HTTP endpoint /metrics on the listening port. +In your case, {{ template "kube-state-metrics.fullname" . }}.{{ template "kube-state-metrics.namespace" . }}.svc.cluster.local:{{ .Values.service.port }}/metrics + +They are served either as plaintext or protobuf depending on the Accept header. +They are designed to be consumed either by Prometheus itself or by a scraper that is compatible with scraping a Prometheus client endpoint. + diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/_helpers.tpl new file mode 100755 index 000000000..6ae0e647f --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/_helpers.tpl @@ -0,0 +1,47 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "kube-state-metrics.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kube-state-metrics.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "kube-state-metrics.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "kube-state-metrics.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-state-metrics.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/clusterrole.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/clusterrole.yaml new file mode 100755 index 000000000..319aec16c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/clusterrole.yaml @@ -0,0 +1,180 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + name: {{ template "kube-state-metrics.fullname" . }} +rules: +{{ if .Values.collectors.certificatesigningrequests }} +- apiGroups: ["certificates.k8s.io"] + resources: + - certificatesigningrequests + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.configmaps }} +- apiGroups: [""] + resources: + - configmaps + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.cronjobs }} +- apiGroups: ["batch"] + resources: + - cronjobs + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.daemonsets }} +- apiGroups: ["extensions", "apps"] + resources: + - daemonsets + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.deployments }} +- apiGroups: ["extensions", "apps"] + resources: + - deployments + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.endpoints }} +- apiGroups: [""] + resources: + - endpoints + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.horizontalpodautoscalers }} +- apiGroups: ["autoscaling"] + resources: + - horizontalpodautoscalers + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.ingresses }} +- apiGroups: ["extensions", "networking.k8s.io"] + resources: + - ingresses + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.jobs }} +- apiGroups: ["batch"] + resources: + - jobs + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.limitranges }} +- apiGroups: [""] + resources: + - limitranges + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.mutatingwebhookconfigurations }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - mutatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.namespaces }} +- apiGroups: [""] + resources: + - namespaces + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.networkpolicies }} +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.nodes }} +- apiGroups: [""] + resources: + - nodes + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.persistentvolumeclaims }} +- apiGroups: [""] + resources: + - persistentvolumeclaims + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.persistentvolumes }} +- apiGroups: [""] + resources: + - persistentvolumes + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.poddisruptionbudgets }} +- apiGroups: ["policy"] + resources: + - poddisruptionbudgets + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.pods }} +- apiGroups: [""] + resources: + - pods + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.replicasets }} +- apiGroups: ["extensions", "apps"] + resources: + - replicasets + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.replicationcontrollers }} +- apiGroups: [""] + resources: + - replicationcontrollers + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.resourcequotas }} +- apiGroups: [""] + resources: + - resourcequotas + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.secrets }} +- apiGroups: [""] + resources: + - secrets + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.services }} +- apiGroups: [""] + resources: + - services + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.statefulsets }} +- apiGroups: ["apps"] + resources: + - statefulsets + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.storageclasses }} +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.validatingwebhookconfigurations }} +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - validatingwebhookconfigurations + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.volumeattachments }} +- apiGroups: ["storage.k8s.io"] + resources: + - volumeattachments + verbs: ["list", "watch"] +{{ end -}} +{{ if .Values.collectors.verticalpodautoscalers }} +- apiGroups: ["autoscaling.k8s.io"] + resources: + - verticalpodautoscalers + verbs: ["list", "watch"] +{{ end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/clusterrolebinding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..4635985aa --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + name: {{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-state-metrics.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/deployment.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/deployment.yaml new file mode 100755 index 000000000..2574e05ea --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/deployment.yaml @@ -0,0 +1,192 @@ +apiVersion: apps/v1 +{{- if .Values.autosharding.enabled }} +kind: StatefulSet +{{- else }} +kind: Deployment +{{- end }} +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} + replicas: {{ .Values.replicas }} +{{- if .Values.autosharding.enabled }} + serviceName: {{ template "kube-state-metrics.fullname" . }} + volumeClaimTemplates: [] +{{- end }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} + app.kubernetes.io/instance: "{{ .Release.Name }}" +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 8 }} +{{- end }} +{{- if .Values.podAnnotations }} + annotations: +{{ toYaml .Values.podAnnotations | indent 8 }} +{{- end }} + spec: + hostNetwork: {{ .Values.hostNetwork }} + serviceAccountName: {{ template "kube-state-metrics.serviceAccountName" . }} + {{- if .Values.securityContext.enabled }} + securityContext: + fsGroup: {{ .Values.securityContext.fsGroup }} + runAsGroup: {{ .Values.securityContext.runAsGroup }} + runAsUser: {{ .Values.securityContext.runAsUser }} + runAsNonRoot: {{ .Values.securityContext.runAsNonRoot }} + {{- end }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + containers: + - name: {{ .Chart.Name }} +{{- if .Values.autosharding.enabled }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{{- end }} + args: +{{ if .Values.collectors.certificatesigningrequests }} + - --collectors=certificatesigningrequests +{{ end }} +{{ if .Values.collectors.configmaps }} + - --collectors=configmaps +{{ end }} +{{ if .Values.collectors.cronjobs }} + - --collectors=cronjobs +{{ end }} +{{ if .Values.collectors.daemonsets }} + - --collectors=daemonsets +{{ end }} +{{ if .Values.collectors.deployments }} + - --collectors=deployments +{{ end }} +{{ if .Values.collectors.endpoints }} + - --collectors=endpoints +{{ end }} +{{ if .Values.collectors.horizontalpodautoscalers }} + - --collectors=horizontalpodautoscalers +{{ end }} +{{ if .Values.collectors.ingresses }} + - --collectors=ingresses +{{ end }} +{{ if .Values.collectors.jobs }} + - --collectors=jobs +{{ end }} +{{ if .Values.collectors.limitranges }} + - --collectors=limitranges +{{ end }} +{{ if .Values.collectors.mutatingwebhookconfigurations }} + - --collectors=mutatingwebhookconfigurations +{{ end }} +{{ if .Values.collectors.namespaces }} + - --collectors=namespaces +{{ end }} +{{ if .Values.collectors.networkpolicies }} + - --collectors=networkpolicies +{{ end }} +{{ if .Values.collectors.nodes }} + - --collectors=nodes +{{ end }} +{{ if .Values.collectors.persistentvolumeclaims }} + - --collectors=persistentvolumeclaims +{{ end }} +{{ if .Values.collectors.persistentvolumes }} + - --collectors=persistentvolumes +{{ end }} +{{ if .Values.collectors.poddisruptionbudgets }} + - --collectors=poddisruptionbudgets +{{ end }} +{{ if .Values.collectors.pods }} + - --collectors=pods +{{ end }} +{{ if .Values.collectors.replicasets }} + - --collectors=replicasets +{{ end }} +{{ if .Values.collectors.replicationcontrollers }} + - --collectors=replicationcontrollers +{{ end }} +{{ if .Values.collectors.resourcequotas }} + - --collectors=resourcequotas +{{ end }} +{{ if .Values.collectors.secrets }} + - --collectors=secrets +{{ end }} +{{ if .Values.collectors.services }} + - --collectors=services +{{ end }} +{{ if .Values.collectors.statefulsets }} + - --collectors=statefulsets +{{ end }} +{{ if .Values.collectors.storageclasses }} + - --collectors=storageclasses +{{ end }} +{{ if .Values.collectors.validatingwebhookconfigurations }} + - --collectors=validatingwebhookconfigurations +{{ end }} +{{ if .Values.collectors.verticalpodautoscalers }} + - --collectors=verticalpodautoscalers +{{ end }} +{{ if .Values.collectors.volumeattachments }} + - --collectors=volumeattachments +{{ end }} +{{ if .Values.namespace }} + - --namespace={{ .Values.namespace }} +{{ end }} +{{ if .Values.autosharding.enabled }} + - --pod=$(POD_NAME) + - --pod-namespace=$(POD_NAMESPACE) +{{ end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}" + ports: + - containerPort: 8080 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 5 + timeoutSeconds: 5 +{{- if .Values.resources }} + resources: +{{ toYaml .Values.resources | indent 10 }} +{{- end }} +{{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 8 }} +{{- end }} +{{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{ toYaml .Values.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{ toYaml .Values.tolerations | indent 8 }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/pdb.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/pdb.yaml new file mode 100755 index 000000000..6adb50d79 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/pdb.yaml @@ -0,0 +1,17 @@ +{{- if .Values.podDisruptionBudget -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} +{{ toYaml .Values.podDisruptionBudget | indent 2 }} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/podsecuritypolicy.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/podsecuritypolicy.yaml new file mode 100755 index 000000000..e822ba0e7 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/podsecuritypolicy.yaml @@ -0,0 +1,42 @@ +{{- if .Values.podSecurityPolicy.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + labels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Values.podSecurityPolicy.annotations }} + annotations: +{{ toYaml .Values.podSecurityPolicy.annotations | indent 4 }} +{{- end }} +spec: + privileged: false + volumes: + - 'secret' +{{- if .Values.podSecurityPolicy.additionalVolumes }} +{{ toYaml .Values.podSecurityPolicy.additionalVolumes | indent 4 }} +{{- end }} + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/psp-clusterrole.yaml new file mode 100755 index 000000000..217abc950 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/psp-clusterrole.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.podSecurityPolicy.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml new file mode 100755 index 000000000..feb97f228 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.podSecurityPolicy.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + name: psp-{{ template "kube-state-metrics.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/service.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/service.yaml new file mode 100755 index 000000000..5dacf5217 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/service.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" +{{- if .Values.customLabels }} +{{ toYaml .Values.customLabels | indent 4 }} +{{- end }} + annotations: + {{- if .Values.prometheusScrape }} + prometheus.io/scrape: '{{ .Values.prometheusScrape }}' + {{- end }} + {{- if .Values.service.annotations }} + {{- toYaml .Values.service.annotations | nindent 4 }} + {{- end }} +spec: + type: "{{ .Values.service.type }}" + ports: + - name: "http" + protocol: TCP + port: {{ .Values.service.port }} + {{- if .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: 8080 +{{- if .Values.service.loadBalancerIP }} + loadBalancerIP: "{{ .Values.service.loadBalancerIP }}" +{{- end }} + selector: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/serviceaccount.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/serviceaccount.yaml new file mode 100755 index 000000000..2e8a1ee38 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- if .Values.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.serviceAccount.annotations | indent 4 }} +{{- end }} +imagePullSecrets: +{{ toYaml .Values.serviceAccount.imagePullSecrets | indent 2 }} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/servicemonitor.yaml new file mode 100755 index 000000000..54cde362d --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/servicemonitor.yaml @@ -0,0 +1,25 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + {{- if .Values.prometheus.monitor.additionalLabels }} +{{ toYaml .Values.prometheus.monitor.additionalLabels | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + endpoints: + - port: http + {{- if .Values.prometheus.monitor.honorLabels }} + honorLabels: true + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/stsdiscovery-role.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/stsdiscovery-role.yaml new file mode 100755 index 000000000..9770b0498 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/stsdiscovery-role.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - apps + resourceNames: + - {{ template "kube-state-metrics.fullname" . }} + resources: + - statefulsets + verbs: + - get + - list + - watch +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml new file mode 100755 index 000000000..6a2e5bfe7 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.autosharding.enabled .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} + labels: + app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "kube-state-metrics.fullname" . }} + namespace: {{ template "kube-state-metrics.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/values.yaml new file mode 100755 index 000000000..654f19144 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kube-state-metrics/values.yaml @@ -0,0 +1,156 @@ +# Default values for kube-state-metrics. +prometheusScrape: true +image: + repository: rancher/mirrored-coreos-kube-state-metrics + tag: v1.9.7 + pullPolicy: IfNotPresent + +imagePullSecrets: [] +# - name: "image-pull-secret" + +# If set to true, this will deploy kube-state-metrics as a StatefulSet and the data +# will be automatically sharded across <.Values.replicas> pods using the built-in +# autodiscovery feature: https://github.com/kubernetes/kube-state-metrics#automated-sharding +# This is an experimental feature and there are no stability guarantees. +autosharding: + enabled: false + +replicas: 1 + +service: + port: 8080 + # Default to clusterIP for backward compatibility + type: ClusterIP + nodePort: 0 + loadBalancerIP: "" + annotations: {} + +customLabels: {} + +hostNetwork: false + +rbac: + # If true, create & use RBAC resources + create: true + +serviceAccount: + # Specifies whether a ServiceAccount should be created, require rbac true + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + # Reference to one or more secrets to be used when pulling images + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + imagePullSecrets: [] + # ServiceAccount annotations. + # Use case: AWS EKS IAM roles for service accounts + # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html + annotations: {} + +prometheus: + monitor: + enabled: false + additionalLabels: {} + namespace: "" + honorLabels: false + +## Specify if a Pod Security Policy for kube-state-metrics must be created +## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ +## +podSecurityPolicy: + enabled: false + annotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + additionalVolumes: [] + +securityContext: + enabled: true + runAsNonRoot: true + runAsGroup: 65534 + runAsUser: 65534 + fsGroup: 65534 + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + +## Affinity settings for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ +affinity: {} + +## Tolerations for pod assignment +## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: [] + +# Annotations to be added to the pod +podAnnotations: {} + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +# Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ +podDisruptionBudget: {} + +# Available collectors for kube-state-metrics. By default all available +# collectors are enabled. +collectors: + certificatesigningrequests: true + configmaps: true + cronjobs: true + daemonsets: true + deployments: true + endpoints: true + horizontalpodautoscalers: true + ingresses: true + jobs: true + limitranges: true + mutatingwebhookconfigurations: true + namespaces: true + networkpolicies: true + nodes: true + persistentvolumeclaims: true + persistentvolumes: true + poddisruptionbudgets: true + pods: true + replicasets: true + replicationcontrollers: true + resourcequotas: true + secrets: true + services: true + statefulsets: true + storageclasses: true + validatingwebhookconfigurations: true + verticalpodautoscalers: false + volumeattachments: true + +# Namespace to be enabled for collecting resources. By default all namespaces are collected. +# namespace: "" + +## Override the deployment namespace +## +namespaceOverride: "" + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 64Mi + # requests: + # cpu: 10m + # memory: 32Mi + +## Provide a k8s version to define apiGroups for podSecurityPolicy Cluster Role. +## For example: kubeTargetVersionOverride: 1.14.9 +## +kubeTargetVersionOverride: "" diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/.helmignore new file mode 100755 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/Chart.yaml new file mode 100755 index 000000000..a82ef1d32 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/Chart.yaml @@ -0,0 +1,13 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: kubeAdmControllerManager +type: application +version: 0.1.3 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/README.md new file mode 100755 index 000000000..dcecc69da --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/README.md @@ -0,0 +1,54 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/_helpers.tpl new file mode 100755 index 000000000..f77b8edf4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/_helpers.tpl @@ -0,0 +1,87 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) .Release.Namespace (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +release: {{ .Release.Name | quote }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100755 index 000000000..95346dee6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,74 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml new file mode 100755 index 000000000..ed78792e5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,135 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + - --metrics-addr=$(PORT) + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: PORT + value: :{{ .Values.clients.port }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100755 index 000000000..a3509c160 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml new file mode 100755 index 000000000..571e13138 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100755 index 000000000..2f3d7e54c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,39 @@ +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: + - port: metrics + proxyUrl: {{ template "pushProxy.proxyUrl" . }} + {{- if .Values.clients.https.enabled }} + params: + _scheme: [https] + {{- end }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ include "pushProxy.client.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/values.yaml new file mode 100755 index 000000000..e1bcf79a5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmControllerManager/values.yaml @@ -0,0 +1,86 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Note: if using a cloud provider, the nodes that contain the PushProxy client must allow the node(s) that contain(s) +# the PushProxy proxy to communicate with it on port {{ .Values.clients.port }}. If you have special restrictions, +# (i.e. client should only run on etcd nodes and only control plane should have access to the port on the etcd node), +# you will need to set the clients / proxy nodeSelector and tolerations accordingly + +# Configuration + +global: + cattle: + systemDefaultRegistry: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-client + tag: v0.1.0-rancher1-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.0-rancher1-proxy + command: ["pushprox-proxy"] \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/.helmignore new file mode 100755 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/Chart.yaml new file mode 100755 index 000000000..bfb047ae6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/Chart.yaml @@ -0,0 +1,13 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: kubeAdmEtcd +type: application +version: 0.1.3 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/README.md new file mode 100755 index 000000000..dcecc69da --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/README.md @@ -0,0 +1,54 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/_helpers.tpl new file mode 100755 index 000000000..f77b8edf4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/_helpers.tpl @@ -0,0 +1,87 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) .Release.Namespace (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +release: {{ .Release.Name | quote }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml new file mode 100755 index 000000000..95346dee6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,74 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-clients.yaml new file mode 100755 index 000000000..ed78792e5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,135 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + - --metrics-addr=$(PORT) + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: PORT + value: :{{ .Values.clients.port }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100755 index 000000000..a3509c160 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml new file mode 100755 index 000000000..571e13138 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml new file mode 100755 index 000000000..2f3d7e54c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,39 @@ +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: + - port: metrics + proxyUrl: {{ template "pushProxy.proxyUrl" . }} + {{- if .Values.clients.https.enabled }} + params: + _scheme: [https] + {{- end }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ include "pushProxy.client.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/values.yaml new file mode 100755 index 000000000..e1bcf79a5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmEtcd/values.yaml @@ -0,0 +1,86 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Note: if using a cloud provider, the nodes that contain the PushProxy client must allow the node(s) that contain(s) +# the PushProxy proxy to communicate with it on port {{ .Values.clients.port }}. If you have special restrictions, +# (i.e. client should only run on etcd nodes and only control plane should have access to the port on the etcd node), +# you will need to set the clients / proxy nodeSelector and tolerations accordingly + +# Configuration + +global: + cattle: + systemDefaultRegistry: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-client + tag: v0.1.0-rancher1-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.0-rancher1-proxy + command: ["pushprox-proxy"] \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/.helmignore new file mode 100755 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/Chart.yaml new file mode 100755 index 000000000..ffe9ae70c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/Chart.yaml @@ -0,0 +1,13 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: kubeAdmProxy +type: application +version: 0.1.3 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/README.md new file mode 100755 index 000000000..dcecc69da --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/README.md @@ -0,0 +1,54 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/_helpers.tpl new file mode 100755 index 000000000..f77b8edf4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/_helpers.tpl @@ -0,0 +1,87 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) .Release.Namespace (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +release: {{ .Release.Name | quote }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml new file mode 100755 index 000000000..95346dee6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,74 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-clients.yaml new file mode 100755 index 000000000..ed78792e5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-clients.yaml @@ -0,0 +1,135 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + - --metrics-addr=$(PORT) + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: PORT + value: :{{ .Values.clients.port }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml new file mode 100755 index 000000000..a3509c160 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-proxy.yaml new file mode 100755 index 000000000..571e13138 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml new file mode 100755 index 000000000..2f3d7e54c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,39 @@ +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: + - port: metrics + proxyUrl: {{ template "pushProxy.proxyUrl" . }} + {{- if .Values.clients.https.enabled }} + params: + _scheme: [https] + {{- end }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ include "pushProxy.client.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/values.yaml new file mode 100755 index 000000000..e1bcf79a5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmProxy/values.yaml @@ -0,0 +1,86 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Note: if using a cloud provider, the nodes that contain the PushProxy client must allow the node(s) that contain(s) +# the PushProxy proxy to communicate with it on port {{ .Values.clients.port }}. If you have special restrictions, +# (i.e. client should only run on etcd nodes and only control plane should have access to the port on the etcd node), +# you will need to set the clients / proxy nodeSelector and tolerations accordingly + +# Configuration + +global: + cattle: + systemDefaultRegistry: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-client + tag: v0.1.0-rancher1-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.0-rancher1-proxy + command: ["pushprox-proxy"] \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/.helmignore new file mode 100755 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/Chart.yaml new file mode 100755 index 000000000..794197de1 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/Chart.yaml @@ -0,0 +1,13 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: kubeAdmScheduler +type: application +version: 0.1.3 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/README.md new file mode 100755 index 000000000..dcecc69da --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/README.md @@ -0,0 +1,54 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/_helpers.tpl new file mode 100755 index 000000000..f77b8edf4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/_helpers.tpl @@ -0,0 +1,87 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) .Release.Namespace (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +release: {{ .Release.Name | quote }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml new file mode 100755 index 000000000..95346dee6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,74 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-clients.yaml new file mode 100755 index 000000000..ed78792e5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,135 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + - --metrics-addr=$(PORT) + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: PORT + value: :{{ .Values.clients.port }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100755 index 000000000..a3509c160 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml new file mode 100755 index 000000000..571e13138 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml new file mode 100755 index 000000000..2f3d7e54c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,39 @@ +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: + - port: metrics + proxyUrl: {{ template "pushProxy.proxyUrl" . }} + {{- if .Values.clients.https.enabled }} + params: + _scheme: [https] + {{- end }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ include "pushProxy.client.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/values.yaml new file mode 100755 index 000000000..e1bcf79a5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/kubeAdmScheduler/values.yaml @@ -0,0 +1,86 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Note: if using a cloud provider, the nodes that contain the PushProxy client must allow the node(s) that contain(s) +# the PushProxy proxy to communicate with it on port {{ .Values.clients.port }}. If you have special restrictions, +# (i.e. client should only run on etcd nodes and only control plane should have access to the port on the etcd node), +# you will need to set the clients / proxy nodeSelector and tolerations accordingly + +# Configuration + +global: + cattle: + systemDefaultRegistry: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-client + tag: v0.1.0-rancher1-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.0-rancher1-proxy + command: ["pushprox-proxy"] \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/.helmignore new file mode 100755 index 000000000..f0c131944 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/Chart.yaml new file mode 100755 index 000000000..9c0a1b590 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/Chart.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +appVersion: v0.7.0 +description: A Helm chart for k8s prometheus adapter +home: https://github.com/DirectXMan12/k8s-prometheus-adapter +keywords: +- hpa +- metrics +- prometheus +- adapter +maintainers: +- email: mattias.gees@jetstack.io + name: mattiasgees +- name: steven-sheehy +- email: hfernandez@mesosphere.com + name: hectorj2f +name: prometheus-adapter +sources: +- https://github.com/kubernetes/charts +- https://github.com/DirectXMan12/k8s-prometheus-adapter +version: 2.7.1 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/README.md new file mode 100755 index 000000000..1fe1fad66 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/README.md @@ -0,0 +1,147 @@ +# Prometheus Adapter + +Installs the [Prometheus Adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter) for the Custom Metrics API. Custom metrics are used in Kubernetes by [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to scale workloads based upon your own metric pulled from an external metrics provider like Prometheus. This chart complements the [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server) chart that provides resource only metrics. + +## Prerequisites + +Kubernetes 1.14+ + +## Get Repo Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +# Helm 3 +$ helm install [RELEASE_NAME] prometheus-community/prometheus-adapter + +# Helm 2 +$ helm install --name [RELEASE_NAME] prometheus-community/prometheus-adapter +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +# Helm 3 +$ helm uninstall [RELEASE_NAME] + +# Helm 2 +# helm delete --purge [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +# Helm 3 or 2 +$ helm upgrade [RELEASE_NAME] [CHART] --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +## Configuration + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +# Helm 2 +$ helm inspect values prometheus-community/prometheus-adapter + +# Helm 3 +$ helm show values prometheus-community/prometheus-adapter +``` + +### Prometheus Service Endpoint + +To use the chart, ensure the `prometheus.url` and `prometheus.port` are configured with the correct Prometheus service endpoint. If Prometheus is exposed under HTTPS the host's CA Bundle must be exposed to the container using `extraVolumes` and `extraVolumeMounts`. + +### Adapter Rules + +Additionally, the chart comes with a set of default rules out of the box but they may pull in too many metrics or not map them correctly for your needs. Therefore, it is recommended to populate `rules.custom` with a list of rules (see the [config document](https://github.com/DirectXMan12/k8s-prometheus-adapter/blob/master/docs/config.md) for the proper format). + +### Horizontal Pod Autoscaler Metrics + +Finally, to configure your Horizontal Pod Autoscaler to use the custom metric, see the custom metrics section of the [HPA walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics). + +The Prometheus Adapter can serve three different [metrics APIs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-metrics-apis): + +### Custom Metrics + +Enabling this option will cause custom metrics to be served at `/apis/custom.metrics.k8s.io/v1beta1`. Enabled by default when `rules.default` is true, but can be customized by populating `rules.custom`: + +```yaml +rules: + custom: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_custom_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### External Metrics + +Enabling this option will cause external metrics to be served at `/apis/external.metrics.k8s.io/v1beta1`. Can be enabled by populating `rules.external`: + +```yaml +rules: + external: + - seriesQuery: '{__name__=~"^some_metric_count$"}' + resources: + template: <<.Resource>> + name: + matches: "" + as: "my_external_metric" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) +``` + +### Resource Metrics + +Enabling this option will cause resource metrics to be served at `/apis/metrics.k8s.io/v1beta1`. Resource metrics will allow pod CPU and Memory metrics to be used in [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) as well as the `kubectl top` command. Can be enabled by populating `rules.resource`: + +```yaml +rules: + resource: + cpu: + containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m])) by (<<.GroupBy>>) + nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>) + resources: + overrides: + instance: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + memory: + containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>) + nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>) + resources: + overrides: + instance: + resource: node + namespace: + resource: namespace + pod: + resource: pod + containerLabel: container + window: 3m +``` + +**NOTE:** Setting a value for `rules.resource` will also deploy the resource metrics API service, providing the same functionality as [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server). As such it is not possible to deploy them both in the same cluster. diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/NOTES.txt b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/NOTES.txt new file mode 100755 index 000000000..b7b9b9932 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/NOTES.txt @@ -0,0 +1,9 @@ +{{ template "k8s-prometheus-adapter.fullname" . }} has been deployed. +In a few minutes you should be able to list metrics using the following command(s): +{{ if .Values.rules.resource }} + kubectl get --raw /apis/metrics.k8s.io/v1beta1 +{{- end }} + kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1 +{{ if .Values.rules.external }} + kubectl get --raw /apis/external.metrics.k8s.io/v1beta1 +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/_helpers.tpl new file mode 100755 index 000000000..5d0b993c5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/_helpers.tpl @@ -0,0 +1,43 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "k8s-prometheus-adapter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "k8s-prometheus-adapter.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "k8s-prometheus-adapter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "k8s-prometheus-adapter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "k8s-prometheus-adapter.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml new file mode 100755 index 000000000..2bc9eb740 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-system-auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-auth-reader-role-binding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-auth-reader-role-binding.yaml new file mode 100755 index 000000000..60f18f2b3 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-auth-reader-role-binding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-deployment.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-deployment.yaml new file mode 100755 index 000000000..2659d9aa7 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-deployment.yaml @@ -0,0 +1,125 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + app: {{ template "k8s-prometheus-adapter.name" . }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.podLabels }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + name: {{ template "k8s-prometheus-adapter.name" . }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/custom-metrics-configmap.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - /adapter + - --secure-port={{ .Values.listenPort }} + {{- if .Values.tls.enable }} + - --tls-cert-file=/var/run/serving-cert/tls.crt + - --tls-private-key-file=/var/run/serving-cert/tls.key + {{- end }} + - --cert-dir=/tmp/cert + - --logtostderr=true + - --prometheus-url={{ .Values.prometheus.url }}{{ if .Values.prometheus.port }}:{{ .Values.prometheus.port }}{{end}}{{ .Values.prometheus.path }} + - --metrics-relist-interval={{ .Values.metricsRelistInterval }} + - --v={{ .Values.logLevel }} + - --config=/etc/adapter/config.yaml + ports: + - containerPort: {{ .Values.listenPort }} + name: https + livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + readinessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 30 + {{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["all"] + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | trim | nindent 8 }} + {{ end }} + - mountPath: /etc/adapter/ + name: config + readOnly: true + - mountPath: /tmp + name: tmp + {{- if .Values.tls.enable }} + - mountPath: /var/run/serving-cert + name: volume-serving-cert + readOnly: true + {{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{- toYaml .Values.nodeSelector | nindent 8 }} +{{- end }} + affinity: + {{- toYaml .Values.affinity | nindent 8 }} + priorityClassName: {{ .Values.priorityClassName }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{- toYaml .Values.tolerations | nindent 8 }} +{{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + volumes: + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | trim | nindent 6 }} + {{ end }} + - name: config + configMap: + name: {{ .Values.rules.existing | default (include "k8s-prometheus-adapter.fullname" . ) }} + - name: tmp + emptyDir: {} + {{- if .Values.tls.enable }} + - name: volume-serving-cert + secret: + secretName: {{ template "k8s-prometheus-adapter.fullname" . }} + {{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-pdb.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-pdb.yaml new file mode 100755 index 000000000..b70309f6f --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-pdb.yaml @@ -0,0 +1,22 @@ +{{- if .Values.podDisruptionBudget.enabled }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + {{- if .Values.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app: {{ template "k8s-prometheus-adapter.name" . }} + release: {{ .Release.Name }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-psp.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-psp.yaml new file mode 100755 index 000000000..a88c9c2f2 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-psp.yaml @@ -0,0 +1,68 @@ +{{- if .Values.psp.create -}} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + {{- if .Values.hostNetwork.enabled }} + hostNetwork: true + {{- end }} + fsGroup: + rule: RunAsAny + runAsGroup: + rule: RunAsAny + runAsUser: + rule: MustRunAs + ranges: + - min: 1024 + max: 65535 + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - secret + - emptyDir + - configMap +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +rules: +- apiGroups: + - 'policy' + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "k8s-prometheus-adapter.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-psp +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml new file mode 100755 index 000000000..ec7e5e476 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-service-account.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-service-account.yaml new file mode 100755 index 000000000..42ef0267e --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-service-account.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-service.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-service.yaml new file mode 100755 index 000000000..6bccda911 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiserver-service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + ports: + - port: {{ .Values.service.port }} + protocol: TCP + targetPort: https + selector: + app: {{ template "k8s-prometheus-adapter.name" . }} + release: {{ .Release.Name }} + type: {{ .Values.service.type }} + diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml new file mode 100755 index 000000000..1eb380d70 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-apiservice.yaml @@ -0,0 +1,23 @@ +{{- if or .Values.rules.default .Values.rules.custom }} +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: v1beta1.custom.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + {{ if .Values.tls.enable -}} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: custom.metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: {{ if .Values.tls.enable }}false{{ else }}true{{ end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml new file mode 100755 index 000000000..33daf7113 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-cluster-role.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +rules: +- apiGroups: + - custom.metrics.k8s.io + resources: ["*"] + verbs: ["*"] +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-configmap.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-configmap.yaml new file mode 100755 index 000000000..fbc155dc8 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-configmap.yaml @@ -0,0 +1,96 @@ +{{- if not .Values.rules.existing -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + config.yaml: | +{{- if or .Values.rules.default .Values.rules.custom }} + rules: +{{- if .Values.rules.default }} + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: [] + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_seconds_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}' + seriesFilters: + - isNot: ^container_.*_total$ + resources: + overrides: + namespace: + resource: namespace + pod: + resource: pod + name: + matches: ^container_(.*)$ + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container!="POD"}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_total$ + resources: + template: <<.Resource>> + name: + matches: "" + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_seconds_total + resources: + template: <<.Resource>> + name: + matches: ^(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: [] + resources: + template: <<.Resource>> + name: + matches: ^(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) +{{- end -}} +{{- if .Values.rules.custom }} +{{ toYaml .Values.rules.custom | indent 4 }} +{{- end -}} +{{- end -}} +{{- if .Values.rules.external }} + externalRules: +{{ toYaml .Values.rules.external | indent 4 }} +{{- end -}} +{{- if .Values.rules.resource }} + resourceRules: +{{ toYaml .Values.rules.resource | indent 6 }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-resource-reader-cluster-role.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-resource-reader-cluster-role.yaml new file mode 100755 index 000000000..319460a33 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/custom-metrics-resource-reader-cluster-role.yaml @@ -0,0 +1,23 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader +rules: +- apiGroups: + - "" + resources: + - namespaces + - pods + - services + - configmaps + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml new file mode 100755 index 000000000..560b2ffcd --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/external-metrics-apiservice.yaml @@ -0,0 +1,23 @@ +{{- if .Values.rules.external }} +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: v1beta1.external.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + {{ if .Values.tls.enable -}} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: external.metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: {{ if .Values.tls.enable }}false{{ else }}true{{ end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml new file mode 100755 index 000000000..4adbd6537 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/external-metrics-cluster-role.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +rules: +- apiGroups: + - "external.metrics.k8s.io" + resources: + - "*" + verbs: + - list + - get + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/hpa-custom-metrics-cluster-role-binding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/hpa-custom-metrics-cluster-role-binding.yaml new file mode 100755 index 000000000..93ade6f8f --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/hpa-custom-metrics-cluster-role-binding.yaml @@ -0,0 +1,23 @@ +{{- /* +This if must be aligned with custom-metrics-cluster-role.yaml +as otherwise this binding will point to not existing role. +*/ -}} +{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/hpa-external-metrics-cluster-role-binding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/hpa-external-metrics-cluster-role-binding.yaml new file mode 100755 index 000000000..0776029af --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/hpa-external-metrics-cluster-role-binding.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.rbac.create .Values.rules.external -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-external-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics +subjects: +- kind: ServiceAccount + name: horizontal-pod-autoscaler + namespace: kube-system +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml new file mode 100755 index 000000000..f1a619912 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/resource-metrics-apiservice.yaml @@ -0,0 +1,23 @@ +{{- if .Values.rules.resource}} +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: v1beta1.metrics.k8s.io +spec: + service: + name: {{ template "k8s-prometheus-adapter.fullname" . }} + namespace: {{ .Release.Namespace | quote }} + {{ if .Values.tls.enable -}} + caBundle: {{ b64enc .Values.tls.ca }} + {{- end }} + group: metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: {{ if .Values.tls.enable }}false{{ else }}true{{ end }} + groupPriorityMinimum: 100 + versionPriority: 100 +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml new file mode 100755 index 000000000..0534af11e --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml @@ -0,0 +1,19 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +subjects: +- kind: ServiceAccount + name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }} + namespace: {{ .Release.Namespace | quote }} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml new file mode 100755 index 000000000..01a307d69 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/resource-metrics-cluster-role.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.rbac.create .Values.rules.resource -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.name" . }}-metrics +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + verbs: + - get + - list + - watch +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/secret.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/secret.yaml new file mode 100755 index 000000000..38e7cb624 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/templates/secret.yaml @@ -0,0 +1,15 @@ +{{- if .Values.tls.enable -}} +apiVersion: v1 +kind: Secret +metadata: + labels: + app: {{ template "k8s-prometheus-adapter.name" . }} + chart: {{ template "k8s-prometheus-adapter.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + name: {{ template "k8s-prometheus-adapter.fullname" . }} +type: kubernetes.io/tls +data: + tls.crt: {{ b64enc .Values.tls.certificate }} + tls.key: {{ b64enc .Values.tls.key }} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/values.yaml new file mode 100755 index 000000000..fa5234889 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-adapter/values.yaml @@ -0,0 +1,152 @@ +# Default values for k8s-prometheus-adapter.. +affinity: {} + +image: + repository: rancher/mirrored-directxman12-k8s-prometheus-adapter-amd64 + tag: v0.7.0 + pullPolicy: IfNotPresent + +logLevel: 4 + +metricsRelistInterval: 1m + +listenPort: 6443 + +nodeSelector: {} + +priorityClassName: "" + +# Url to access prometheus +prometheus: + url: http://prometheus.default.svc + port: 9090 + path: "" + +replicas: 1 + +rbac: + # Specifies whether RBAC resources should be created + create: true + +psp: + # Specifies whether PSP resources should be created + create: false + +serviceAccount: + # Specifies whether a service account should be created + create: true + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + +resources: {} + # requests: + # cpu: 100m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + +rules: + default: true + custom: [] +# - seriesQuery: '{__name__=~"^some_metric_count$"}' +# resources: +# template: <<.Resource>> +# name: +# matches: "" +# as: "my_custom_metric" +# metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + # Mounts a configMap with pre-generated rules for use. Overrides the + # default, custom, external and resource entries + existing: + external: [] +# - seriesQuery: '{__name__=~"^some_metric_count$"}' +# resources: +# template: <<.Resource>> +# name: +# matches: "" +# as: "my_external_metric" +# metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + resource: {} +# cpu: +# containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m])) by (<<.GroupBy>>) +# nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>) +# resources: +# overrides: +# instance: +# resource: node +# namespace: +# resource: namespace +# pod: +# resource: pod +# containerLabel: container +# memory: +# containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>) +# nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>) +# resources: +# overrides: +# instance: +# resource: node +# namespace: +# resource: namespace +# pod: +# resource: pod +# containerLabel: container +# window: 3m + +service: + annotations: {} + port: 443 + type: ClusterIP + +tls: + enable: false + ca: |- + # Public CA file that signed the APIService + key: |- + # Private key of the APIService + certificate: |- + # Public key of the APIService + +# Any extra volumes +extraVolumes: [] + # - name: example-name + # hostPath: + # path: /path/on/host + # type: DirectoryOrCreate + # - name: ssl-certs + # hostPath: + # path: /etc/ssl/certs/ca-bundle.crt + # type: File + +# Any extra volume mounts +extraVolumeMounts: [] + # - name: example-name + # mountPath: /path/in/container + # - name: ssl-certs + # mountPath: /etc/ssl/certs/ca-certificates.crt + # readOnly: true + +tolerations: [] + +# Labels added to the pod +podLabels: {} + +# Annotations added to the pod +podAnnotations: {} + +hostNetwork: + # Specifies if prometheus-adapter should be started in hostNetwork mode. + # + # You would require this enabled if you use alternate overlay networking for pods and + # API server unable to communicate with metrics-server. As an example, this is required + # if you use Weave network on EKS + enabled: false + +podDisruptionBudget: + # Specifies if PodDisruptionBudget should be enabled + # When enabled, minAvailable or maxUnavailable should also be defined. + enabled: false + minAvailable: + maxUnavailable: 1 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/.helmignore new file mode 100755 index 000000000..f0c131944 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/Chart.yaml new file mode 100755 index 000000000..1c1c87a25 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +appVersion: 1.0.1 +description: A Helm chart for prometheus node-exporter +home: https://github.com/prometheus/node_exporter/ +keywords: +- node-exporter +- prometheus +- exporter +maintainers: +- email: gianrubio@gmail.com + name: gianrubio +- name: vsliouniaev +name: prometheus-node-exporter +sources: +- https://github.com/prometheus/node_exporter/ +version: 1.11.2 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/README.md new file mode 100755 index 000000000..babde05e0 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/README.md @@ -0,0 +1,63 @@ +# Prometheus Node Exporter + +Prometheus exporter for hardware and OS metrics exposed by *NIX kernels, written in Go with pluggable metric collectors. + +This chart bootstraps a prometheus [Node Exporter](http://github.com/prometheus/node_exporter) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Get Repo Info + +```console +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Install Chart + +```console +# Helm 3 +$ helm install [RELEASE_NAME] prometheus-community/prometheus-node-exporter + +# Helm 2 +$ helm install --name [RELEASE_NAME] prometheus-community/prometheus-node-exporter +``` + +_See [configuration](#configuration) below._ + +_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._ + +## Uninstall Chart + +```console +# Helm 3 +$ helm uninstall [RELEASE_NAME] + +# Helm 2 +# helm delete --purge [RELEASE_NAME] +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._ + +## Upgrading Chart + +```console +# Helm 3 or 2 +$ helm upgrade [RELEASE_NAME] [CHART] --install +``` + +_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._ + +## Configuring + +See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments, visit the chart's [values.yaml](./values.yaml), or run these configuration commands: + +```console +# Helm 2 +$ helm inspect values prometheus-community/prometheus-node-exporter + +# Helm 3 +$ helm show values prometheus-community/prometheus-node-exporter +``` diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/NOTES.txt b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/NOTES.txt new file mode 100755 index 000000000..dc272fa99 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/NOTES.txt @@ -0,0 +1,15 @@ +1. Get the application URL by running these commands: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "prometheus-node-exporter.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ template "prometheus-node-exporter.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ template "prometheus-node-exporter.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ template "prometheus-node-exporter.namespace" . }} {{ template "prometheus-node-exporter.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ template "prometheus-node-exporter.namespace" . }} -l "app={{ template "prometheus-node-exporter.name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:9100 to use your application" + kubectl port-forward --namespace {{ template "prometheus-node-exporter.namespace" . }} $POD_NAME 9100 +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/_helpers.tpl new file mode 100755 index 000000000..e8260688f --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/_helpers.tpl @@ -0,0 +1,66 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "prometheus-node-exporter.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "prometheus-node-exporter.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Generate basic labels */}} +{{- define "prometheus-node-exporter.labels" }} +app: {{ template "prometheus-node-exporter.name" . }} +heritage: {{.Release.Service }} +release: {{.Release.Name }} +chart: {{ template "prometheus-node-exporter.chart" . }} +{{- if .Values.podLabels}} +{{ toYaml .Values.podLabels }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-node-exporter.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "prometheus-node-exporter.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "prometheus-node-exporter.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-node-exporter.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/daemonset.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/daemonset.yaml new file mode 100755 index 000000000..5aa3f4ea8 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/daemonset.yaml @@ -0,0 +1,151 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} + labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app: {{ template "prometheus-node-exporter.name" . }} + release: {{ .Release.Name }} + {{- if .Values.updateStrategy }} + updateStrategy: +{{ toYaml .Values.updateStrategy | indent 4 }} + {{- end }} + template: + metadata: + labels: {{ include "prometheus-node-exporter.labels" . | indent 8 }} + {{- if .Values.podAnnotations }} + annotations: + {{- toYaml .Values.podAnnotations | nindent 8 }} + {{- end }} + spec: +{{- if and .Values.rbac.create .Values.serviceAccount.create }} + serviceAccountName: {{ template "prometheus-node-exporter.serviceAccountName" . }} +{{- end }} +{{- if .Values.securityContext }} + securityContext: +{{ toYaml .Values.securityContext | indent 8 }} +{{- end }} +{{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} +{{- end }} + containers: + - name: node-exporter + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + - --web.listen-address=$(HOST_IP):{{ .Values.service.port }} +{{- if .Values.extraArgs }} +{{ toYaml .Values.extraArgs | indent 12 }} +{{- end }} + env: + - name: HOST_IP + {{- if .Values.service.listenOnAllInterfaces }} + value: 0.0.0.0 + {{- else }} + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + {{- end }} + ports: + - name: metrics + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + livenessProbe: + httpGet: + path: / + port: {{ .Values.service.port }} + readinessProbe: + httpGet: + path: / + port: {{ .Values.service.port }} + resources: +{{ toYaml .Values.resources | indent 12 }} + volumeMounts: + - name: proc + mountPath: /host/proc + readOnly: true + - name: sys + mountPath: /host/sys + readOnly: true + {{- if .Values.extraHostVolumeMounts }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- if $mount.mountPropagation }} + mountPropagation: {{ $mount.mountPropagation }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.sidecarVolumeMount }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: true + {{- end }} + {{- end }} + {{- if .Values.configmaps }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + {{- end }} + {{- end }} +{{- if .Values.sidecars }} +{{ toYaml .Values.sidecars | indent 8 }} + {{- if .Values.sidecarVolumeMount }} + volumeMounts: + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} + readOnly: {{ $mount.readOnly }} + {{- end }} + {{- end }} +{{- end }} + hostNetwork: {{ .Values.hostNetwork }} + hostPID: true +{{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 8 }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{- toYaml .Values.Selector | nindent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{- toYaml .Values.tolerations | nindent 8 }} +{{- end }} + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + {{- if .Values.extraHostVolumeMounts }} + {{- range $_, $mount := .Values.extraHostVolumeMounts }} + - name: {{ $mount.name }} + hostPath: + path: {{ $mount.hostPath }} + {{- end }} + {{- end }} + {{- if .Values.sidecarVolumeMount }} + {{- range $_, $mount := .Values.sidecarVolumeMount }} + - name: {{ $mount.name }} + emptyDir: + medium: Memory + {{- end }} + {{- end }} + {{- if .Values.configmaps }} + {{- range $_, $mount := .Values.configmaps }} + - name: {{ $mount.name }} + configMap: + name: {{ $mount.name }} + {{- end }} + {{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/endpoints.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/endpoints.yaml new file mode 100755 index 000000000..8daaeaaff --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/endpoints.yaml @@ -0,0 +1,18 @@ +{{- if .Values.endpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} + labels: +{{ include "prometheus-node-exporter.labels" . | indent 4 }} +subsets: + - addresses: + {{- range .Values.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: metrics + port: 9100 + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/monitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/monitor.yaml new file mode 100755 index 000000000..4e31ba363 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/monitor.yaml @@ -0,0 +1,25 @@ +{{- if .Values.prometheus.monitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} + labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} + {{- if .Values.prometheus.monitor.additionalLabels }} +{{ toYaml .Values.prometheus.monitor.additionalLabels | indent 4 }} + {{- end }} +spec: + selector: + matchLabels: + app: {{ template "prometheus-node-exporter.name" . }} + release: {{ .Release.Name }} + endpoints: + - port: metrics + {{- if .Values.prometheus.monitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.prometheus.monitor.scrapeTimeout }} + {{- end }} +{{- if .Values.prometheus.monitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheus.monitor.relabelings | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml new file mode 100755 index 000000000..cb433369c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/psp-clusterrole.yaml @@ -0,0 +1,15 @@ +{{- if .Values.rbac.create }} +{{- if .Values.rbac.pspEnabled }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: psp-{{ template "prometheus-node-exporter.fullname" . }} + labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "prometheus-node-exporter.fullname" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml new file mode 100755 index 000000000..d36d93ecf --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/psp-clusterrolebinding.yaml @@ -0,0 +1,17 @@ +{{- if .Values.rbac.create }} +{{- if .Values.rbac.pspEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: psp-{{ template "prometheus-node-exporter.fullname" . }} + labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp-{{ template "prometheus-node-exporter.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "prometheus-node-exporter.fullname" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/psp.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/psp.yaml new file mode 100755 index 000000000..f00506c98 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/psp.yaml @@ -0,0 +1,52 @@ +{{- if .Values.rbac.create }} +{{- if .Values.rbac.pspEnabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} + labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} +spec: + privileged: false + # Required to prevent escalations to root. + # allowPrivilegeEscalation: false + # This is redundant with non-root + disallow privilege escalation, + # but we can provide it for defense in depth. + #requiredDropCapabilities: + # - ALL + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + - 'hostPath' + hostNetwork: true + hostIPC: false + hostPID: true + hostPorts: + - min: 0 + max: 65535 + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/service.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/service.yaml new file mode 100755 index 000000000..b0a447fe3 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/service.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "prometheus-node-exporter.fullname" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} +{{- if .Values.service.annotations }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +{{- end }} + labels: {{ include "prometheus-node-exporter.labels" . | indent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + {{- if ( and (eq .Values.service.type "NodePort" ) (not (empty .Values.service.nodePort)) ) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + targetPort: {{ .Values.service.targetPort }} + protocol: TCP + name: metrics + selector: + app: {{ template "prometheus-node-exporter.name" . }} + release: {{ .Release.Name }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/serviceaccount.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/serviceaccount.yaml new file mode 100755 index 000000000..bd1c223b4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/templates/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if .Values.rbac.create -}} +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "prometheus-node-exporter.serviceAccountName" . }} + namespace: {{ template "prometheus-node-exporter.namespace" . }} + labels: + app: {{ template "prometheus-node-exporter.name" . }} + chart: {{ template "prometheus-node-exporter.chart" . }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +imagePullSecrets: +{{ toYaml .Values.serviceAccount.imagePullSecrets | indent 2 }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/values.yaml new file mode 100755 index 000000000..2dc21acf3 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/prometheus-node-exporter/values.yaml @@ -0,0 +1,143 @@ +# Default values for prometheus-node-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. +image: + repository: rancher/mirrored-prom-node-exporter + tag: v1.0.1 + pullPolicy: IfNotPresent + +service: + type: ClusterIP + port: 9100 + targetPort: 9100 + nodePort: + listenOnAllInterfaces: true + annotations: + prometheus.io/scrape: "true" + +prometheus: + monitor: + enabled: false + additionalLabels: {} + namespace: "" + + relabelings: [] + scrapeTimeout: 10s + +## Customize the updateStrategy if set +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 200m + # memory: 50Mi + # requests: + # cpu: 100m + # memory: 30Mi + +serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + imagePullSecrets: [] + +securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + +rbac: + ## If true, create & use RBAC resources + ## + create: true + ## If true, create & use Pod Security Policy resources + ## https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + pspEnabled: true + +# for deployments that have node_exporter deployed outside of the cluster, list +# their addresses here +endpoints: [] + +# Expose the service to the host network +hostNetwork: true + +## Assign a group of affinity scheduling rules +## +affinity: {} +# nodeAffinity: +# requiredDuringSchedulingIgnoredDuringExecution: +# nodeSelectorTerms: +# - matchFields: +# - key: metadata.name +# operator: In +# values: +# - target-host-name + +# Annotations to be added to node exporter pods +podAnnotations: {} + +# Extra labels to be added to node exporter pods +podLabels: {} + +## Assign a nodeSelector if operating a hybrid cluster +## +nodeSelector: {} +# beta.kubernetes.io/arch: amd64 +# beta.kubernetes.io/os: linux + +tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + +## Assign a PriorityClassName to pods if set +# priorityClassName: "" + +## Additional container arguments +## +extraArgs: [] +# - --collector.diskstats.ignored-devices=^(ram|loop|fd|(h|s|v)d[a-z]|nvme\\d+n\\d+p)\\d+$ +# - --collector.textfile.directory=/run/prometheus + +## Additional mounts from the host +## +extraHostVolumeMounts: [] +# - name: +# hostPath: +# mountPath: +# readOnly: true|false +# mountPropagation: None|HostToContainer|Bidirectional + +## Additional configmaps to be mounted. +## +configmaps: [] +# - name: +# mountPath: + +## Override the deployment namespace +## +namespaceOverride: "" + +## Additional containers for export metrics to text file +## +sidecars: [] +## - name: nvidia-dcgm-exporter +## image: nvidia/dcgm-exporter:1.4.3 + +## Volume for sidecar containers +## +sidecarVolumeMount: [] +## - name: collector-textfiles +## mountPath: /run/prometheus +## readOnly: false diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/.helmignore new file mode 100755 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/Chart.yaml new file mode 100755 index 000000000..e5205567e --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/Chart.yaml @@ -0,0 +1,13 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: rke2ControllerManager +type: application +version: 0.1.3 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/README.md new file mode 100755 index 000000000..dcecc69da --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/README.md @@ -0,0 +1,54 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/_helpers.tpl new file mode 100755 index 000000000..f77b8edf4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/_helpers.tpl @@ -0,0 +1,87 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) .Release.Namespace (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +release: {{ .Release.Name | quote }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100755 index 000000000..95346dee6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,74 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-clients.yaml new file mode 100755 index 000000000..ed78792e5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,135 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + - --metrics-addr=$(PORT) + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: PORT + value: :{{ .Values.clients.port }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100755 index 000000000..a3509c160 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-proxy.yaml new file mode 100755 index 000000000..571e13138 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100755 index 000000000..2f3d7e54c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,39 @@ +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: + - port: metrics + proxyUrl: {{ template "pushProxy.proxyUrl" . }} + {{- if .Values.clients.https.enabled }} + params: + _scheme: [https] + {{- end }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ include "pushProxy.client.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/values.yaml new file mode 100755 index 000000000..e1bcf79a5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2ControllerManager/values.yaml @@ -0,0 +1,86 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Note: if using a cloud provider, the nodes that contain the PushProxy client must allow the node(s) that contain(s) +# the PushProxy proxy to communicate with it on port {{ .Values.clients.port }}. If you have special restrictions, +# (i.e. client should only run on etcd nodes and only control plane should have access to the port on the etcd node), +# you will need to set the clients / proxy nodeSelector and tolerations accordingly + +# Configuration + +global: + cattle: + systemDefaultRegistry: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-client + tag: v0.1.0-rancher1-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.0-rancher1-proxy + command: ["pushprox-proxy"] \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/.helmignore new file mode 100755 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/Chart.yaml new file mode 100755 index 000000000..7320aec04 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/Chart.yaml @@ -0,0 +1,13 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: rke2Etcd +type: application +version: 0.1.3 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/README.md new file mode 100755 index 000000000..dcecc69da --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/README.md @@ -0,0 +1,54 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/_helpers.tpl new file mode 100755 index 000000000..f77b8edf4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/_helpers.tpl @@ -0,0 +1,87 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) .Release.Namespace (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +release: {{ .Release.Name | quote }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml new file mode 100755 index 000000000..95346dee6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,74 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-clients.yaml new file mode 100755 index 000000000..ed78792e5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-clients.yaml @@ -0,0 +1,135 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + - --metrics-addr=$(PORT) + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: PORT + value: :{{ .Values.clients.port }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml new file mode 100755 index 000000000..a3509c160 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-proxy.yaml new file mode 100755 index 000000000..571e13138 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-proxy.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml new file mode 100755 index 000000000..2f3d7e54c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,39 @@ +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: + - port: metrics + proxyUrl: {{ template "pushProxy.proxyUrl" . }} + {{- if .Values.clients.https.enabled }} + params: + _scheme: [https] + {{- end }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ include "pushProxy.client.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/values.yaml new file mode 100755 index 000000000..e1bcf79a5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Etcd/values.yaml @@ -0,0 +1,86 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Note: if using a cloud provider, the nodes that contain the PushProxy client must allow the node(s) that contain(s) +# the PushProxy proxy to communicate with it on port {{ .Values.clients.port }}. If you have special restrictions, +# (i.e. client should only run on etcd nodes and only control plane should have access to the port on the etcd node), +# you will need to set the clients / proxy nodeSelector and tolerations accordingly + +# Configuration + +global: + cattle: + systemDefaultRegistry: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-client + tag: v0.1.0-rancher1-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.0-rancher1-proxy + command: ["pushprox-proxy"] \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/.helmignore new file mode 100755 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/Chart.yaml new file mode 100755 index 000000000..1e2201169 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/Chart.yaml @@ -0,0 +1,13 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: rke2Proxy +type: application +version: 0.1.3 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/README.md new file mode 100755 index 000000000..dcecc69da --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/README.md @@ -0,0 +1,54 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/_helpers.tpl new file mode 100755 index 000000000..f77b8edf4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/_helpers.tpl @@ -0,0 +1,87 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) .Release.Namespace (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +release: {{ .Release.Name | quote }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml new file mode 100755 index 000000000..95346dee6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,74 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-clients.yaml new file mode 100755 index 000000000..ed78792e5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-clients.yaml @@ -0,0 +1,135 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + - --metrics-addr=$(PORT) + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: PORT + value: :{{ .Values.clients.port }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml new file mode 100755 index 000000000..a3509c160 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-proxy.yaml new file mode 100755 index 000000000..571e13138 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-proxy.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml new file mode 100755 index 000000000..2f3d7e54c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,39 @@ +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: + - port: metrics + proxyUrl: {{ template "pushProxy.proxyUrl" . }} + {{- if .Values.clients.https.enabled }} + params: + _scheme: [https] + {{- end }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ include "pushProxy.client.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/values.yaml new file mode 100755 index 000000000..e1bcf79a5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Proxy/values.yaml @@ -0,0 +1,86 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Note: if using a cloud provider, the nodes that contain the PushProxy client must allow the node(s) that contain(s) +# the PushProxy proxy to communicate with it on port {{ .Values.clients.port }}. If you have special restrictions, +# (i.e. client should only run on etcd nodes and only control plane should have access to the port on the etcd node), +# you will need to set the clients / proxy nodeSelector and tolerations accordingly + +# Configuration + +global: + cattle: + systemDefaultRegistry: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-client + tag: v0.1.0-rancher1-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.0-rancher1-proxy + command: ["pushprox-proxy"] \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/.helmignore new file mode 100755 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/Chart.yaml new file mode 100755 index 000000000..4b076fb84 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/Chart.yaml @@ -0,0 +1,13 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: rke2Scheduler +type: application +version: 0.1.3 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/README.md new file mode 100755 index 000000000..dcecc69da --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/README.md @@ -0,0 +1,54 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/_helpers.tpl new file mode 100755 index 000000000..f77b8edf4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/_helpers.tpl @@ -0,0 +1,87 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) .Release.Namespace (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +release: {{ .Release.Name | quote }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml new file mode 100755 index 000000000..95346dee6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,74 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-clients.yaml new file mode 100755 index 000000000..ed78792e5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-clients.yaml @@ -0,0 +1,135 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + - --metrics-addr=$(PORT) + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: PORT + value: :{{ .Values.clients.port }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml new file mode 100755 index 000000000..a3509c160 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-proxy.yaml new file mode 100755 index 000000000..571e13138 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml new file mode 100755 index 000000000..2f3d7e54c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,39 @@ +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: + - port: metrics + proxyUrl: {{ template "pushProxy.proxyUrl" . }} + {{- if .Values.clients.https.enabled }} + params: + _scheme: [https] + {{- end }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ include "pushProxy.client.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/values.yaml new file mode 100755 index 000000000..e1bcf79a5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rke2Scheduler/values.yaml @@ -0,0 +1,86 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Note: if using a cloud provider, the nodes that contain the PushProxy client must allow the node(s) that contain(s) +# the PushProxy proxy to communicate with it on port {{ .Values.clients.port }}. If you have special restrictions, +# (i.e. client should only run on etcd nodes and only control plane should have access to the port on the etcd node), +# you will need to set the clients / proxy nodeSelector and tolerations accordingly + +# Configuration + +global: + cattle: + systemDefaultRegistry: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-client + tag: v0.1.0-rancher1-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.0-rancher1-proxy + command: ["pushprox-proxy"] \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/.helmignore new file mode 100755 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/Chart.yaml new file mode 100755 index 000000000..09ef21031 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/Chart.yaml @@ -0,0 +1,13 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: rkeControllerManager +type: application +version: 0.1.3 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/README.md new file mode 100755 index 000000000..dcecc69da --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/README.md @@ -0,0 +1,54 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/_helpers.tpl new file mode 100755 index 000000000..f77b8edf4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/_helpers.tpl @@ -0,0 +1,87 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) .Release.Namespace (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +release: {{ .Release.Name | quote }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml new file mode 100755 index 000000000..95346dee6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,74 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-clients.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-clients.yaml new file mode 100755 index 000000000..ed78792e5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-clients.yaml @@ -0,0 +1,135 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + - --metrics-addr=$(PORT) + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: PORT + value: :{{ .Values.clients.port }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml new file mode 100755 index 000000000..a3509c160 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-proxy.yaml new file mode 100755 index 000000000..571e13138 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-proxy.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml new file mode 100755 index 000000000..2f3d7e54c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,39 @@ +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: + - port: metrics + proxyUrl: {{ template "pushProxy.proxyUrl" . }} + {{- if .Values.clients.https.enabled }} + params: + _scheme: [https] + {{- end }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ include "pushProxy.client.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/values.yaml new file mode 100755 index 000000000..e1bcf79a5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeControllerManager/values.yaml @@ -0,0 +1,86 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Note: if using a cloud provider, the nodes that contain the PushProxy client must allow the node(s) that contain(s) +# the PushProxy proxy to communicate with it on port {{ .Values.clients.port }}. If you have special restrictions, +# (i.e. client should only run on etcd nodes and only control plane should have access to the port on the etcd node), +# you will need to set the clients / proxy nodeSelector and tolerations accordingly + +# Configuration + +global: + cattle: + systemDefaultRegistry: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-client + tag: v0.1.0-rancher1-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.0-rancher1-proxy + command: ["pushprox-proxy"] \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/.helmignore new file mode 100755 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/Chart.yaml new file mode 100755 index 000000000..a4f4b02e4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/Chart.yaml @@ -0,0 +1,13 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: rkeEtcd +type: application +version: 0.1.3 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/README.md new file mode 100755 index 000000000..dcecc69da --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/README.md @@ -0,0 +1,54 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/_helpers.tpl new file mode 100755 index 000000000..f77b8edf4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/_helpers.tpl @@ -0,0 +1,87 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) .Release.Namespace (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +release: {{ .Release.Name | quote }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml new file mode 100755 index 000000000..95346dee6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,74 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-clients.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-clients.yaml new file mode 100755 index 000000000..ed78792e5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-clients.yaml @@ -0,0 +1,135 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + - --metrics-addr=$(PORT) + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: PORT + value: :{{ .Values.clients.port }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml new file mode 100755 index 000000000..a3509c160 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-proxy.yaml new file mode 100755 index 000000000..571e13138 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-proxy.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml new file mode 100755 index 000000000..2f3d7e54c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,39 @@ +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: + - port: metrics + proxyUrl: {{ template "pushProxy.proxyUrl" . }} + {{- if .Values.clients.https.enabled }} + params: + _scheme: [https] + {{- end }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ include "pushProxy.client.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/values.yaml new file mode 100755 index 000000000..e1bcf79a5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeEtcd/values.yaml @@ -0,0 +1,86 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Note: if using a cloud provider, the nodes that contain the PushProxy client must allow the node(s) that contain(s) +# the PushProxy proxy to communicate with it on port {{ .Values.clients.port }}. If you have special restrictions, +# (i.e. client should only run on etcd nodes and only control plane should have access to the port on the etcd node), +# you will need to set the clients / proxy nodeSelector and tolerations accordingly + +# Configuration + +global: + cattle: + systemDefaultRegistry: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-client + tag: v0.1.0-rancher1-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.0-rancher1-proxy + command: ["pushprox-proxy"] \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/.helmignore new file mode 100755 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/Chart.yaml new file mode 100755 index 000000000..f86115b68 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/Chart.yaml @@ -0,0 +1,13 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: rkeProxy +type: application +version: 0.1.3 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/README.md new file mode 100755 index 000000000..dcecc69da --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/README.md @@ -0,0 +1,54 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/_helpers.tpl new file mode 100755 index 000000000..f77b8edf4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/_helpers.tpl @@ -0,0 +1,87 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) .Release.Namespace (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +release: {{ .Release.Name | quote }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-clients-rbac.yaml new file mode 100755 index 000000000..95346dee6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,74 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-clients.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-clients.yaml new file mode 100755 index 000000000..ed78792e5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-clients.yaml @@ -0,0 +1,135 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + - --metrics-addr=$(PORT) + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: PORT + value: :{{ .Values.clients.port }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml new file mode 100755 index 000000000..a3509c160 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-proxy.yaml new file mode 100755 index 000000000..571e13138 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-proxy.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-servicemonitor.yaml new file mode 100755 index 000000000..2f3d7e54c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,39 @@ +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: + - port: metrics + proxyUrl: {{ template "pushProxy.proxyUrl" . }} + {{- if .Values.clients.https.enabled }} + params: + _scheme: [https] + {{- end }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ include "pushProxy.client.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/values.yaml new file mode 100755 index 000000000..e1bcf79a5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeProxy/values.yaml @@ -0,0 +1,86 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Note: if using a cloud provider, the nodes that contain the PushProxy client must allow the node(s) that contain(s) +# the PushProxy proxy to communicate with it on port {{ .Values.clients.port }}. If you have special restrictions, +# (i.e. client should only run on etcd nodes and only control plane should have access to the port on the etcd node), +# you will need to set the clients / proxy nodeSelector and tolerations accordingly + +# Configuration + +global: + cattle: + systemDefaultRegistry: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-client + tag: v0.1.0-rancher1-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.0-rancher1-proxy + command: ["pushprox-proxy"] \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/.helmignore new file mode 100755 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/Chart.yaml new file mode 100755 index 000000000..9b58f56a5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/Chart.yaml @@ -0,0 +1,13 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/os: linux + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-pushprox +apiVersion: v1 +appVersion: 0.1.0 +description: Sets up a deployment of the PushProx proxy and a DaemonSet of PushProx + clients. +name: rkeScheduler +type: application +version: 0.1.3 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/README.md new file mode 100755 index 000000000..dcecc69da --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/README.md @@ -0,0 +1,54 @@ +# rancher-pushprox + +A Rancher chart based on Rancher [PushProx](https://github.com/rancher/PushProx) that sets up a Deployment of a PushProx proxy and a DaemonSet of PushProx clients on a Kubernetes cluster. + +Installs [rancher-pushprox](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-pushprox) to create PushProx clients that can access their host's network and register with a PushProx proxy. A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR is also included that is configured to scrape the metrics from each of the clients through the proxy. + +Using an instance of this chart is suitable for the following scenarios: +- You need to scrape metrics from a port that should not be accessible outside of the host (e.g. scraping `etcd` metrics in a hardened cluster) +- You need to scrape metrics on a host that are not exposed outside of 127.0.0.1 (e.g. scraping `kube-proxy` metrics) +- You need to scrape metrics through HTTPS using certs hosted directly on `hostPath` +- You need to scrape metrics from Kubernetes components that require authorization via a service account (e.g. permissions to make request to `/metrics`) +- You need to scrape metrics without access to cacerts (i.e. enable `insecureSkipVerify`) + +The clients and proxy are created based on a Rancher fork of the [prometheus-community/PushProx](https://github.com/prometheus-community/PushProx) project. + +## Configuration + +The following tables list the configurable parameters of the rancher-pushprox chart and their default values. + +### General + +#### Required +| Parameter | Description | Example | +| ----- | ----------- | ------ | +| `component` | The component that is being monitored | `kube-etcd` +| `metricsPort` | The port on the host that contains the metrics you want to scrape (e.g. `http://:/metrics`) | `2379` | + +#### Optional +| Parameter | Description | Default | +| ----- | ----------- | ------ | +| `serviceMonitor.enabled` | Deploys a [Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#servicemonitor) ServiceMonitor CR that is configured to scrape metrics on the hosts that the clients are deployed on via the proxy. Also deploys a Service that points to all pods with the expected client name that exposes the `metricsPort` selected | `true` | +| `clients.enabled` | Deploys a DaemonSet of clients that are each capable of scraping endpoints on the hostNetwork it is deployed on | `true` | +| `clients.port` | The port where the client will publish PushProx client-specific metrics. If deploying multiple clients onto the same node, the clients should not have conflicting ports | `9369` | +| `clients.proxyUrl` | Overrides the default proxyUrl setting of `http://pushprox-{{ .Values.component }}-proxy.{{ . Release.Namespace }}.svc.cluster.local:{{ .Values.proxy.port }}"` with the `proxyUrl` specified | `""` | +| `clients.useLocalhost` | Sets a flag on each client deployment to redirect scrapes directed to `HOST_IP` to `127.0.0.1` | `false` | +| `clients.https.enabled` | Enables scraping metrics via HTTPS using the provided TLS certs that exist on each host | `false` | +| `clients.https.useServiceAccountCredentials` | If set to true, the client will create a service account with permissions to scrape `/metrics` endpoint of Kubernetes components. The client will use the service account token provided to make authorized scrape requests to the Kubernetes API | `false` | +| `clients.https.insecureSkipVerify` | If set to true, the client will disable SSL security checks | `false` | +| `clients.https.certDir` | A `hostPath` where TLS certs can be found. This path is mounted as a volume on an `initContainer` which copies only the necessary files over to an EmptyDir volume used by each client. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.certFile` | The path to the TLS cert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.keyFile` | The path to the TLS key file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.https.caCertFile` | The path to the TLS cacert file located within `clients.https.certDir`. Required and only used if `clients.https.enabled` is set | `""` | +| `clients.resources` | Set resource limits and requests for the client container | `{}` | +| `clients.nodeSelector` | Select which nodes to deploy the clients on | `{}` | +| `clients.tolerations` | Specify tolerations for clients | `[]` | +| `proxy.enabled` | Deploys the proxy that each client will register with | `true` | +| `proxy.port` | The port exposed by the proxy that each client will register with to allow metrics to be scraped from the host | `8080` | +| `proxy.resources` | Set resource limits and requests for the proxy container | `{}` | +| `proxy.nodeSelector` | Select which nodes the proxy can be deployed on | `{}` | +| `proxy.tolerations` | Specify tolerations (if necessary) to allow the proxy to be deployed on the selected node | `[]` | + +*Tip: The filepaths set in `clients.https.File` can include wildcard characters*. + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for examples of how this chart can be used. \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/_helpers.tpl new file mode 100755 index 000000000..f77b8edf4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/_helpers.tpl @@ -0,0 +1,87 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# General + +{{- define "pushprox.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{- define "pushProxy.commonLabels" -}} +release: {{ .Release.Name }} +component: {{ .Values.component | quote }} +provider: kubernetes +{{- end -}} + +{{- define "pushProxy.proxyUrl" -}} +{{- $_ := (required "Template requires either .Values.proxy.port or .Values.client.proxyUrl to set proxyUrl for client" (or .Values.clients.proxyUrl .Values.proxy.port)) -}} +{{- if .Values.clients.proxyUrl -}} +{{ printf "%s" .Values.clients.proxyUrl }} +{{- else -}} +{{ printf "http://%s.%s.svc:%d" (include "pushProxy.proxy.name" .) .Release.Namespace (int .Values.proxy.port) }} +{{- end -}}{{- end -}} + +# Client + +{{- define "pushProxy.client.name" -}} +{{- printf "pushprox-%s-client" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.client.labels" -}} +k8s-app: {{ template "pushProxy.client.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# Proxy + +{{- define "pushProxy.proxy.name" -}} +{{- printf "pushprox-%s-proxy" (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.proxy.labels" -}} +k8s-app: {{ template "pushProxy.proxy.name" . }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} + +# ServiceMonitor + +{{- define "pushprox.serviceMonitor.name" -}} +{{- printf "%s-%s" .Release.Name (required ".Values.component is required" .Values.component) -}} +{{- end -}} + +{{- define "pushProxy.serviceMonitor.labels" -}} +app: {{ template "pushprox.serviceMonitor.name" . }} +release: {{ .Release.Name | quote }} +{{ template "pushProxy.commonLabels" . }} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml new file mode 100755 index 000000000..95346dee6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-clients-rbac.yaml @@ -0,0 +1,74 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.client.name" . }} +{{- if and .Values.clients.https.enabled .Values.clients.https.useServiceAccountCredentials }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.client.name" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.client.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: true + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + - 'emptyDir' + - 'hostPath' + allowedHostPaths: + - pathPrefix: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + readOnly: true +{{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-clients.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-clients.yaml new file mode 100755 index 000000000..ed78792e5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-clients.yaml @@ -0,0 +1,135 @@ +{{- if .Values.clients }}{{- if .Values.clients.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} + pushprox-exporter: "client" +spec: + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.client.labels" . | nindent 8 }} + spec: + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.clients.tolerations }} +{{ toYaml .Values.clients.tolerations | indent 8 }} +{{- end }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + serviceAccountName: {{ template "pushProxy.client.name" . }} + containers: + - name: pushprox-client + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: + {{- range .Values.clients.command }} + - {{ . | quote }} + {{- end }} + args: + - --fqdn=$(HOST_IP) + - --proxy-url=$(PROXY_URL) + - --metrics-addr=$(PORT) + - --allow-port={{ required "Need .Values.metricsPort to configure client to be allowed to scrape metrics at port" .Values.metricsPort}} + {{- if .Values.clients.useLocalhost }} + - --use-localhost + {{- end }} + {{- if .Values.clients.https.enabled }} + {{- if .Values.clients.https.insecureSkipVerify }} + - --insecure-skip-verify + {{- end }} + {{- if .Values.clients.https.useServiceAccountCredentials }} + - --token-path=/var/run/secrets/kubernetes.io/serviceaccount/token + {{- end }} + {{- if .Values.clients.https.certDir }} + - --tls.cert=/etc/ssl/push-proxy/push-proxy.pem + - --tls.key=/etc/ssl/push-proxy/push-proxy-key.pem + - --tls.cacert=/etc/ssl/push-proxy/push-proxy-ca-cert.pem + {{- end }} + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: PORT + value: :{{ .Values.clients.port }} + - name: PROXY_URL + value: {{ template "pushProxy.proxyUrl" . }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + volumeMounts: + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + {{- end }} + {{- if .Values.clients.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} + {{- end }} + {{- if and .Values.clients.https.enabled .Values.clients.https.certDir }} + initContainers: + - name: copy-certs + image: {{ template "system_default_registry" . }}{{ .Values.clients.copyCertsImage.repository }}:{{ .Values.clients.copyCertsImage.tag }} + command: + - sh + - -c + - | + echo "Searching for files to copy within the source volume" + echo "cert: ${CERT_FILE_NAME}" + echo "key: ${KEY_FILE_NAME}" + echo "cacert: ${CACERT_FILE_NAME}" + + CERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CERT_FILE_NAME}" | sort -r | head -n 1) + KEY_FILE_SOURCE=$(find /etc/source/ -type f -name "${KEY_FILE_NAME}" | sort -r | head -n 1) + CACERT_FILE_SOURCE=$(find /etc/source/ -type f -name "${CACERT_FILE_NAME}" | sort -r | head -n 1) + + test -z ${CERT_FILE_SOURCE} && echo "Failed to find cert file" && exit 1 + test -z ${KEY_FILE_SOURCE} && echo "Failed to find key file" && exit 1 + test -z ${CACERT_FILE_SOURCE} && echo "Failed to find cacert file" && exit 1 + + echo "Copying cert file from $CERT_FILE_SOURCE to $CERT_FILE_TARGET" + cp $CERT_FILE_SOURCE $CERT_FILE_TARGET || exit 1 + chmod 444 $CERT_FILE_TARGET || exit 1 + + echo "Copying key file from $KEY_FILE_SOURCE to $KEY_FILE_TARGET" + cp $KEY_FILE_SOURCE $KEY_FILE_TARGET || exit 1 + chmod 444 $KEY_FILE_TARGET || exit 1 + + echo "Copying cacert file from $CACERT_FILE_SOURCE to $CACERT_FILE_TARGET" + cp $CACERT_FILE_SOURCE $CACERT_FILE_TARGET || exit 1 + chmod 444 $CACERT_FILE_TARGET || exit 1 + env: + - name: CERT_FILE_NAME + value: {{ required "Need a TLS cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.certFile }} + - name: KEY_FILE_NAME + value: {{ required "Need a TLS key file for scraping metrics endpoint over HTTPs" .Values.clients.https.keyFile }} + - name: CACERT_FILE_NAME + value: {{ required "Need a TLS CA cert file for scraping metrics endpoint over HTTPs" .Values.clients.https.caCertFile }} + - name: CERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy.pem + - name: KEY_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-key.pem + - name: CACERT_FILE_TARGET + value: /etc/ssl/push-proxy/push-proxy-ca-cert.pem + securityContext: + runAsNonRoot: false + volumeMounts: + - name: metrics-cert-dir-source + mountPath: /etc/source + readOnly: true + - name: metrics-cert-dir + mountPath: /etc/ssl/push-proxy + volumes: + - name: metrics-cert-dir-source + hostPath: + path: {{ required "Need access to volume on host with the SSL cert files to use HTTPs" .Values.clients.https.certDir }} + - name: metrics-cert-dir + emptyDir: {} + {{- end }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml new file mode 100755 index 000000000..a3509c160 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-proxy-rbac.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +rules: +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "pushProxy.proxy.name" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "pushProxy.proxy.name" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "pushProxy.proxy.name" . }} +subjects: + - kind: ServiceAccount + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ .Release.Namespace }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-proxy.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-proxy.yaml new file mode 100755 index 000000000..571e13138 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-proxy.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.proxy }}{{ if .Values.proxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} + pushprox-exporter: "proxy" +spec: + selector: + matchLabels: {{ include "pushProxy.proxy.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "pushProxy.proxy.labels" . | nindent 8 }} + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.proxy.nodeSelector }} +{{ toYaml .Values.proxy.nodeSelector | indent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.proxy.tolerations }} +{{ toYaml .Values.proxy.tolerations | indent 8 }} +{{- end }} + serviceAccountName: {{ template "pushProxy.proxy.name" . }} + containers: + - name: pushprox-proxy + image: {{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }} + command: + {{- range .Values.proxy.command }} + - {{ . | quote }} + {{- end }} + {{- if .Values.proxy.resources }} + resources: {{ toYaml .Values.proxy.resources | nindent 10 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.proxy.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +spec: + ports: + - name: pp-proxy + port: {{ required "Need .Values.proxy.port to configure proxy" .Values.proxy.port }} + protocol: TCP + targetPort: {{ .Values.proxy.port }} + selector: {{ include "pushProxy.proxy.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml new file mode 100755 index 000000000..2f3d7e54c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/templates/pushprox-servicemonitor.yaml @@ -0,0 +1,39 @@ +{{- if .Values.serviceMonitor }}{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "pushprox.serviceMonitor.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.serviceMonitor.labels" . | nindent 4 }} +spec: + endpoints: + - port: metrics + proxyUrl: {{ template "pushProxy.proxyUrl" . }} + {{- if .Values.clients.https.enabled }} + params: + _scheme: [https] + {{- end }} + jobLabel: component + podTargetLabels: + - component + - pushprox-exporter + namespaceSelector: + matchNames: + - {{ template "pushprox.namespace" . }} + selector: + matchLabels: {{ include "pushProxy.client.labels" . | nindent 6 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "pushProxy.client.name" . }} + namespace: {{ template "pushprox.namespace" . }} + labels: {{ include "pushProxy.client.labels" . | nindent 4 }} +spec: + ports: + - name: metrics + port: {{ required "Need .Values.metricsPort to configure client to listen to metrics at port" .Values.metricsPort}} + protocol: TCP + targetPort: {{ .Values.metricsPort }} + selector: {{ include "pushProxy.client.labels" . | nindent 4 }} +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/values.yaml new file mode 100755 index 000000000..e1bcf79a5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/rkeScheduler/values.yaml @@ -0,0 +1,86 @@ +# Default values for rancher-pushprox. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Default image containing both the proxy and the client was generated from the following Dockerfile +# https://github.com/prometheus-community/PushProx/blob/eeadbe766641699129920ccfaaaa30a85c67fe81/Dockerfile#L1-L15 + +# Note: if using a cloud provider, the nodes that contain the PushProxy client must allow the node(s) that contain(s) +# the PushProxy proxy to communicate with it on port {{ .Values.clients.port }}. If you have special restrictions, +# (i.e. client should only run on etcd nodes and only control plane should have access to the port on the etcd node), +# you will need to set the clients / proxy nodeSelector and tolerations accordingly + +# Configuration + +global: + cattle: + systemDefaultRegistry: "" + +# The component that is being monitored (i.e. etcd) +component: "component" + +# The port containing the metrics that need to be scraped +metricsPort: 2739 + +# Configure ServiceMonitor that monitors metrics from the metricsPort endpoint +serviceMonitor: + enabled: true + +clients: + enabled: true + # The port which the PushProx client will post PushProx metrics to + port: 9369 + # If unset, this will default to the URL for the proxy service: http://pushprox-{{component}}-proxy.{{namepsace}}.svc.cluster.local:{{proxy.port}} + # Should be modified if the clients are being deployed outside the cluster where the proxy rests, otherwise leave it null + proxyUrl: "" + # If set to true, the client will forward any requests from the host IP to 127.0.0.1 + # It will only allow proxy requests to the metricsPort specified + useLocalhost: false + # Configuration for accessing metrics via HTTPS + https: + # Does the client require https to access the metrics? + enabled: false + # If set to true, the client will create a service account with adequate permissions and set a flag + # on the client to use the service account token provided by it to make authorized scrape requests + useServiceAccountCredentials: false + # If set to true, the client will disable SSL security checks + insecureSkipVerify: false + # Directory on host where necessary TLS cert and key to scrape metrics can be found + certDir: "" + # Filenames for files located in .Values.clients.https.certDir that correspond to TLS settings + certFile: "" + keyFile: "" + caCertFile: "" + + # Resource limits + resources: {} + + # Options to select all nodes to deploy client DaemonSet on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-client + tag: v0.1.0-rancher1-client + command: ["pushprox-client"] + + copyCertsImage: + repository: rancher/mirrored-library-busybox + tag: 1.31.1 + +proxy: + enabled: true + # The port through which PushProx clients will communicate to the proxy + port: 8080 + + # Resource limits + resources: {} + + # Options to select a node to run a single proxy deployment on + nodeSelector: {} + tolerations: [] + + image: + repository: rancher/pushprox-proxy + tag: v0.1.0-rancher1-proxy + command: ["pushprox-proxy"] \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/.helmignore b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/.helmignore new file mode 100755 index 000000000..0e8a0eb36 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/Chart.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/Chart.yaml new file mode 100755 index 000000000..fba9162f2 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/hidden: "true" + catalog.cattle.io/os: windows + catalog.rancher.io/certified: rancher + catalog.rancher.io/namespace: cattle-monitoring-system + catalog.rancher.io/release-name: rancher-windows-exporter +apiVersion: v1 +appVersion: 0.0.4 +description: Sets up monitoring metrics from Windows nodes via Prometheus windows-exporter +maintainers: +- email: arvind.iyengar@rancher.com + name: aiyengar2 +name: windowsExporter +type: application +version: 0.1.0 diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/README.md b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/README.md new file mode 100755 index 000000000..6115b6f25 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/README.md @@ -0,0 +1,17 @@ +# rancher-windows-exporter + +A Rancher chart based on the [prometheus-community/windows-exporter](https://github.com/prometheus-community/windows_exporter) project (previously called wmi-exporter) that sets up a DaemonSet of clients that can scrape windows-exporter metrics from Windows nodes on a Kubernetes cluster. + +A [Prometheus Operator](https://github.com/coreos/prometheus-operator) ServiceMonitor CR and PrometheusRule CR are also created by this chart to collect metrics and add some recording rules to map `windows_` series with their OS-agnostic counterparts. + +## Node Requirements + +Since Windows does not support privileged pods, this chart expects a Named Pipe (`\\.\pipe\rancher_wins`) to exist on the Windows host that allows containers to communicate with the host. This is done by deploying a [rancher/wins](https://github.com/rancher/wins) server on the host. + +The image used by the chart, [windows_exporter-package](https://github.com/rancher/windows_exporter-package), is configured to create a wins client that communicates with the wins server, alongside a running copy of a particular version of [windows-exporter](https://github.com/prometheus-community/windows_exporter). Through the wins client and wins server, the windows-exporter is able to communicate directly with the Windows host to collect metrics and expose them. + +If the cluster you are installing this chart on is a custom cluster that was created via RKE1 with Windows Support enabled, your nodes should already have the wins server running; this should have been added as part of [the bootstrapping process for adding the Windows node onto your RKE1 cluster](https://github.com/rancher/rancher/blob/master/package/windows/bootstrap.ps1). + +## Configuration + +See [rancher-monitoring](https://github.com/rancher/charts/tree/gh-pages/packages/rancher-monitoring) for an example of how this chart can be used. diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/scripts/copy-binary.ps1 b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/scripts/copy-binary.ps1 new file mode 100755 index 000000000..a9a28df96 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/scripts/copy-binary.ps1 @@ -0,0 +1,40 @@ +$ErrorActionPreference = 'Stop' + +function Create-Directory +{ + param ( + [parameter(Mandatory = $false, ValueFromPipeline = $true)] [string]$Path + ) + + if (Test-Path -Path $Path) { + if (-not (Test-Path -Path $Path -PathType Container)) { + # clean the same path file + Remove-Item -Recurse -Force -Path $Path -ErrorAction Ignore | Out-Null + } + + return + } + + New-Item -Force -ItemType Directory -Path $Path | Out-Null +} + +function Transfer-File +{ + param ( + [parameter(Mandatory = $true)] [string]$Src, + [parameter(Mandatory = $true)] [string]$Dst + ) + + if (Test-Path -PathType leaf -Path $Dst) { + $dstHasher = Get-FileHash -Path $Dst + $srcHasher = Get-FileHash -Path $Src + if ($dstHasher.Hash -eq $srcHasher.Hash) { + return + } + } + + $null = Copy-Item -Force -Path $Src -Destination $Dst +} + +Create-Directory -Path "c:\host\etc\windows-exporter" +Transfer-File -Src "c:\etc\windows-exporter\windows-exporter.exe" -Dst "c:\host\etc\windows-exporter\windows-exporter.exe" \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/scripts/proxy-entry.ps1 b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/scripts/proxy-entry.ps1 new file mode 100755 index 000000000..9d0581b66 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/scripts/proxy-entry.ps1 @@ -0,0 +1,11 @@ +# default +$listenPort = "9796" + +if ($env:LISTEN_PORT) { + $listenPort = $env:LISTEN_PORT +} + +# format "UDP:4789 TCP:8080" +$winsPublish = $('TCP:{0}' -f $listenPort) + +wins.exe cli proxy --publish $winsPublish diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/_helpers.tpl new file mode 100755 index 000000000..7365b52f5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/_helpers.tpl @@ -0,0 +1,64 @@ +# Rancher + +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# General + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "windowsExporter.name" -}} +{{ printf "%s-windows-exporter" .Release.Name }} +{{- end -}} + +{{- define "windowsExporter.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride -}} +{{- end -}} + +{{- define "windowsExporter.labels" -}} +k8s-app: {{ template "windowsExporter.name" . }} +release: {{ .Release.Name }} +component: "windows-exporter" +provider: kubernetes +{{- end -}} + +# Client + +{{- define "windowsExporter.client.nodeSelector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: windows +{{- else -}} +kubernetes.io/os: windows +{{- end -}} +{{- if .Values.clients.nodeSelector }} +{{ toYaml .Values.clients.nodeSelector }} +{{- end }} +{{- end -}} + +{{- define "windowsExporter.client.tolerations" -}} +{{- if .Values.clients.tolerations -}} +{{ toYaml .Values.clients.tolerations }} +{{- else -}} +- operator: Exists +{{- end -}} +{{- end -}} + +{{- define "windowsExporter.client.env" -}} +- name: LISTEN_PORT + value: {{ required "Need .Values.clients.port to figure out where to get metrics from" .Values.clients.port | quote }} +{{- if .Values.clients.enabledCollectors }} +- name: ENABLED_COLLECTORS + value: {{ .Values.clients.enabledCollectors | quote }} +{{- end }} +{{- if .Values.clients.env }} +{{ toYaml .Values.clients.env }} +{{- end }} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/configmap.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/configmap.yaml new file mode 100755 index 000000000..5957552de --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/configmap.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "windowsExporter.name" . }}-scripts + namespace: {{ template "windowsExporter.namespace" . }} + labels: {{ include "windowsExporter.labels" . | nindent 4 }} +data: + copy-binary.ps1: |- +{{ .Files.Get "scripts/copy-binary.ps1" | indent 4 }} + proxy-entry.ps1: |- +{{ .Files.Get "scripts/proxy-entry.ps1" | indent 4 }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/daemonset.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/daemonset.yaml new file mode 100755 index 000000000..0d65eb51d --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/daemonset.yaml @@ -0,0 +1,69 @@ +{{- if .Values.clients }}{{ if .Values.clients.enabled }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ template "windowsExporter.name" . }} + namespace: {{ template "windowsExporter.namespace" . }} + labels: {{ include "windowsExporter.labels" . | nindent 4 }} +spec: + selector: + matchLabels: {{ include "windowsExporter.labels" . | nindent 6 }} + template: + metadata: + labels: {{ include "windowsExporter.labels" . | nindent 8 }} + spec: + nodeSelector: {{ include "windowsExporter.client.nodeSelector" . | nindent 8 }} + tolerations: {{ include "windowsExporter.client.tolerations" . | nindent 8 }} + serviceAccountName: {{ template "windowsExporter.name" . }} + containers: + - name: exporter-node-proxy + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: ["pwsh", "-f", "c:/scripts/proxy-entry.ps1"] + ports: + - name: http + containerPort: {{ required "Need .Values.clients.port to figure out where to get metrics from" .Values.clients.port }} + env: {{ include "windowsExporter.client.env" . | nindent 10 }} +{{- if .Values.resources }} + resources: {{ toYaml .Values.clients.proxy.resources | nindent 10 }} +{{- end }} + volumeMounts: + - name: wins-pipe-proxy + mountPath: \\.\pipe\rancher_wins_proxy + - name: exporter-scripts + mountPath: c:/scripts/ + - name: exporter-node + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} +{{- if .Values.clients.args }} + args: {{ .Values.clients.args }} +{{- end }} + env: {{ include "windowsExporter.client.env" . | nindent 10 }} +{{- if .Values.resources }} + resources: {{ toYaml .Values.clients.resources | nindent 10 }} +{{- end }} + volumeMounts: + - name: wins-pipe + mountPath: \\.\pipe\rancher_wins + initContainers: + - name: exporter-node-binary-copy + image: {{ template "system_default_registry" . }}{{ .Values.clients.image.repository }}:{{ .Values.clients.image.tag }} + command: ["pwsh", "-f", "c:/scripts/copy-binary.ps1"] + volumeMounts: + - name: binary-host-path + mountPath: c:/host/etc/windows-exporter + - name: exporter-scripts + mountPath: c:/scripts/ + volumes: + - name: wins-pipe + hostPath: + path: \\.\pipe\rancher_wins + - name: wins-pipe-proxy + hostPath: + path: \\.\pipe\rancher_wins_proxy + - name: binary-host-path + hostPath: + path: c:/etc/windows-exporter + type: DirectoryOrCreate + - name: exporter-scripts + configMap: + name: {{ template "windowsExporter.name" . }}-scripts +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/rbac.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/rbac.yaml new file mode 100755 index 000000000..ebec8f235 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/rbac.yaml @@ -0,0 +1,78 @@ +{{- if .Values.clients }}{{ if .Values.clients.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "windowsExporter.name" . }} + namespace: {{ template "windowsExporter.namespace" . }} + labels: {{ include "windowsExporter.labels" . | nindent 4 }} +rules: +- apiGroups: ['authentication.k8s.io'] + resources: ['tokenreviews'] + verbs: ['create'] +- apiGroups: ['authorization.k8s.io'] + resources: ['subjectaccessreviews'] + verbs: ['create'] +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['{{ template "windowsExporter.name" . }}'] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "windowsExporter.name" . }} + namespace: {{ template "windowsExporter.namespace" . }} + labels: {{ include "windowsExporter.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "windowsExporter.name" . }} +subjects: +- kind: ServiceAccount + name: {{ template "windowsExporter.name" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "windowsExporter.name" . }} + namespace: {{ template "windowsExporter.namespace" . }} + labels: {{ include "windowsExporter.labels" . | nindent 4 }} +{{- if .Values.clients.imagePullSecrets }} +imagePullSecrets: {{ toYaml .Values.clients.imagePullSecrets | nindent 2 }} +{{- end }} +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "windowsExporter.name" . }} + namespace: {{ template "windowsExporter.namespace" . }} + labels: {{ include "windowsExporter.labels" . | nindent 4 }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 0 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' + - 'hostPath' + allowedHostPaths: + - pathPrefix: \\.\pipe\rancher_wins + - pathPrefix: \\.\pipe\rancher_wins_proxy + - pathPrefix: c:/etc/windows-exporter +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/service.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/service.yaml new file mode 100755 index 000000000..944150e5c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/service.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.serviceMonitor .Values.clients }}{{- if and .Values.serviceMonitor.enabled .Values.clients.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "windowsExporter.name" . }} + namespace: {{ template "windowsExporter.namespace" . }} + labels: {{ include "windowsExporter.labels" . | nindent 4 }} +spec: + ports: + - name: windows-metrics + port: {{ required "Need .Values.clients.port to figure out where to get metrics from" .Values.clients.port }} + protocol: TCP + targetPort: {{ .Values.clients.port }} + selector: {{ include "windowsExporter.labels" . | nindent 4 }} +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/servicemonitor.yaml new file mode 100755 index 000000000..368e6282c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/templates/servicemonitor.yaml @@ -0,0 +1,38 @@ +{{- if and .Values.serviceMonitor .Values.clients }}{{- if and .Values.serviceMonitor.enabled .Values.clients.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: {{ include "windowsExporter.labels" . | nindent 4 }} + name: {{ template "windowsExporter.name" . }} + namespace: {{ template "windowsExporter.namespace" . }} +spec: + selector: + matchLabels: {{ include "windowsExporter.labels" . | nindent 6 }} + namespaceSelector: + matchNames: + - {{ template "windowsExporter.namespace" . }} + jobLabel: component + podTargetLabels: + - component + endpoints: + - port: windows-metrics + metricRelabelings: + - sourceLabels: [__name__] + regex: 'wmi_(.*)' + replacement: 'windows_$1' + targetLabel: __name__ + - sourceLabels: [__name__] + regex: windows_mssql_transactions_active_total + replacement: 'windows_mssql_transactions_active' + targetLabel: __name__ + - sourceLabels: [volume, nic] + regex: (.*);(.*) + separator: '' + targetLabel: device + action: replace + replacement: $1$2 + - sourceLabels: [__name__] + regex: windows_cs_logical_processors + replacement: 'system' + targetLabel: mode +{{- end }}{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/values.yaml new file mode 100755 index 000000000..a75fbc34f --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/charts/windowsExporter/values.yaml @@ -0,0 +1,44 @@ +# Default values for rancher-windows-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Configuration + +global: + cattle: + systemDefaultRegistry: "" + +# Configure ServiceMonitor that monitors metrics +serviceMonitor: + enabled: true + +## Components scraping metrics from Windows nodes +## +clients: + enabled: true + + port: 9796 + image: + repository: rancher/windows_exporter-package + tag: v0.0.1 + + # Specify the IP addresses of nodes that you want to collect metrics from + endpoints: [] + + # Get more details on https://github.com/prometheus-community/windows_exporter + args: [] + env: {} + enabledCollectors: "net,os,service,system,cpu,cs,logical_disk,tcp,memory,container" + + # Resource limits + resources: {} + + # Options to select nodes to target for scraping Windows metrics + nodeSelector: {} # Note: {kubernetes.io/os: windows} is default and cannot be overridden + tolerations: [] # Note: if not specified, the default option is to use [{operator: Exists}] + + # Image Pull Secrets for the service account used by the clients + imagePullSecrets: {} + + proxy: + resources: {} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/files/ingress-nginx/nginx.json b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/files/ingress-nginx/nginx.json new file mode 100755 index 000000000..347c9eb05 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/files/ingress-nginx/nginx.json @@ -0,0 +1,1463 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.2.1" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": "${DS_PROMETHEUS}", + "enable": true, + "expr": "sum(changes(nginx_ingress_controller_config_last_reload_successful_timestamp_seconds{instance!=\"unknown\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[30s])) by (controller_class)", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Config Reloads", + "showIn": 0, + "step": "30s", + "tagKeys": "controller_class", + "tags": [], + "titleFormat": "Config Reloaded", + "type": "tags" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1534359654832, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Request Volume", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 82, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(avg_over_time(nginx_ingress_controller_nginx_process_connections{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",state=\"active\"}[2m]))", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Connections", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",status!~\"[4-5].*\"}[2m])) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "95, 99, 99.5", + "title": "Controller Success Rate (non-4|5xx responses)", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 81, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(irate(nginx_ingress_controller_success{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[1m])) * 60", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Config Reloads", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "total" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 83, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(nginx_ingress_controller_config_last_reload_successful{controller_pod=~\"$controller\",controller_namespace=~\"$namespace\"} == 0)", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Last Config Failed", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 3 + }, + "height": "200px", + "id": 86, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress), 0.001)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "network", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Request Volume", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 87, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\",status!~\"[4-5].*\"}[2m])) by (ingress) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Success Rate (non-4|5xx responses)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "height": "200px", + "id": 32, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (irate (nginx_ingress_controller_request_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Received", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (irate (nginx_ingress_controller_response_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Sent", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network I/O pressure", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 77, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(nginx_ingress_controller_nginx_process_resident_memory_bytes{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}) ", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 3, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 10 + }, + "height": "", + "id": 79, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg (rate (nginx_ingress_controller_nginx_process_cpu_seconds_total{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m])) ", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "${DS_PROMETHEUS}", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "hideTimeOverride": false, + "id": 75, + "links": [], + "pageSize": 7, + "repeat": null, + "repeatDirection": "h", + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "Ingress", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "ingress", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Requests", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #A", + "thresholds": [ + "" + ], + "type": "number", + "unit": "ops" + }, + { + "alias": "Errors", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "ops" + }, + { + "alias": "P50 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P90 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P99 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "IN", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #F", + "thresholds": [ + "" + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "OUT", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "Bps" + } + ], + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "D" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }}", + "refId": "E" + }, + { + "expr": "sum(irate(nginx_ingress_controller_request_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "F" + }, + { + "expr": "sum(irate(nginx_ingress_controller_response_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "G" + } + ], + "timeFrom": null, + "title": "Ingress Percentile Response Times and Transfer Rates", + "transform": "table", + "transparent": false, + "type": "table" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "${DS_PROMETHEUS}", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "height": "1024", + "id": 85, + "links": [], + "pageSize": 7, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "TTL", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Current", + "thresholds": [ + "0", + "691200" + ], + "type": "number", + "unit": "s" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "avg(nginx_ingress_controller_ssl_expire_time_seconds{kubernetes_pod_name=~\"$controller\",namespace=~\"$namespace\",ingress=~\"$ingress\"}) by (host) - time()", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ host }}", + "metric": "gke_letsencrypt_cert_expiration", + "refId": "A", + "step": 1 + } + ], + "title": "Ingress Certificate Expiry", + "transform": "timeseries_aggregations", + "type": "table" + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "hide": 0, + "label": "datasource", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "${DS_PROMETHEUS}", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash, controller_namespace)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "${DS_PROMETHEUS}", + "hide": 0, + "includeAll": true, + "label": "Controller Class", + "multi": false, + "name": "controller_class", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\"}, controller_class) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "${DS_PROMETHEUS}", + "hide": 0, + "includeAll": true, + "label": "Controller", + "multi": false, + "name": "controller", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\",controller_class=~\"$controller_class\"}, controller_pod) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "tags": [], + "text": "All", + "value": "$__all" + }, + "datasource": "${DS_PROMETHEUS}", + "hide": 0, + "includeAll": true, + "label": "Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests{namespace=~\"$namespace\",controller_class=~\"$controller_class\",controller=~\"$controller\"}, ingress) ", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Ingress Controller", + "uid": "nginx", + "version": 1 +} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/files/ingress-nginx/request-handling-performance.json b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/files/ingress-nginx/request-handling-performance.json new file mode 100755 index 000000000..5635ae976 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/files/ingress-nginx/request-handling-performance.json @@ -0,0 +1,981 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "6.6.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": 9614, + "graphTooltip": 1, + "id": null, + "iteration": 1582146566338, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "Total time taken for nginx and upstream servers to process a request and send a response", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_request_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total request handling time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The time spent on receiving the response from the upstream server", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 94, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n 0.5,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".5", + "refId": "D" + }, + { + "expr": "histogram_quantile(\n 0.95,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".95", + "refId": "B" + }, + { + "expr": "histogram_quantile(\n 0.99,\n sum by (le)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "legendFormat": ".99", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream response time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 93, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum by (path)(\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "For each path observed, its median upstream response time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 98, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(\n .5,\n sum by (le, path)(\n rate(\n nginx_ingress_controller_response_duration_seconds_bucket{\n ingress =~ \"$ingress\"\n }[1m]\n )\n )\n)", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Median upstream response time by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "Percentage of 4xx and 5xx responses among all responses.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 100, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress = \"$ingress\",\n status =~ \"[4-5].*\"\n}[1m])) / sum by (path) (rate(nginx_ingress_controller_request_duration_seconds_count{\n ingress = \"$ingress\",\n}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error rate by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "For each path observed, the sum of upstream request time", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 102, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (path) (rate(nginx_ingress_controller_response_duration_seconds_sum{ingress = \"$ingress\"}[1m]))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream time consumed by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 101, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " sum (\n rate(\n nginx_ingress_controller_request_duration_seconds_count{\n ingress =~ \"$ingress\",\n status =~\"[4-5].*\",\n }[1m]\n )\n ) by(path, status)\n", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }} {{ status }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response error volume by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 99, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate (\n nginx_ingress_controller_response_size_sum {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path) / sum (\n rate(\n nginx_ingress_controller_response_size_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n) by (path)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ path }}", + "refId": "D" + }, + { + "expr": " sum (rate(nginx_ingress_controller_response_size_bucket{\n namespace =~ \"$namespace\",\n ingress =~ \"$ingress\",\n }[1m])) by (le)\n", + "hide": true, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average response size by Path", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_sum {\n ingress =~ \"$ingress\",\n }[1m]\n)) / sum (\n rate(\n nginx_ingress_controller_ingress_upstream_latency_seconds_count {\n ingress =~ \"$ingress\",\n }[1m]\n )\n)\n", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "average", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream service latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 22, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "hide": 0, + "label": "datasource", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(nginx_ingress_controller_requests, ingress) ", + "hide": 0, + "includeAll": true, + "label": "Service Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests, ingress) ", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX / Request Handling Performance", + "uid": "4GFbkOsZk", + "version": 1 + } diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/files/rancher/rancher-default-home.json b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/files/rancher/rancher-default-home.json new file mode 100755 index 000000000..d2921bf86 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/files/rancher/rancher-default-home.json @@ -0,0 +1,1275 @@ +{ + "annotations": { + "list": [ + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "title": "", + "type": "welcome" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 4 + }, + "height": "180px", + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - avg(irate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "CPU usage (5m avg)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 4 + }, + "height": "180px", + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - sum(node_memory_MemAvailable_bytes{})/ sum(node_memory_MemTotal_bytes{})) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Memory usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 4 + }, + "height": "180px", + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"})\n - sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"})) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Filesystem usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 9 + }, + "height": "1px", + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(1 - (avg(irate(node_cpu_seconds_total{mode=\"idle\"}[5m])))) * sum(kube_node_status_allocatable_cpu_cores{})", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 9 + }, + "height": "1px", + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (kube_node_status_allocatable_cpu_cores{})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 9 + }, + "height": "1px", + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "20%", + "prefix": "", + "prefixFontSize": "20%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(node_memory_MemTotal_bytes{device!~\"rootfs|HarddiskVolume.+\"}) - sum(node_memory_MemAvailable_bytes{device!~\"rootfs|HarddiskVolume.+\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 9 + }, + "height": "1px", + "id": 10, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (kube_node_status_allocatable_memory_bytes{})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 9 + }, + "height": "1px", + "id": 13, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\",})\n - sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 9 + }, + "height": "1px", + "id": 14, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 2051, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1- (avg(irate(node_cpu_seconds_total{mode=\"idle\"}[5m])))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "1- (avg(irate(node_cpu_seconds_total{mode=\"idle\"}[5m])) by (node))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ node }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU usage (5m avg)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 12 + }, + "hiddenSeries": false, + "id": 2052, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (1 - sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "100 * (1- sum(node_memory_MemAvailable_bytes) by (node) / sum(node_memory_MemTotal_bytes) by (node))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ node }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 12 + }, + "hiddenSeries": false, + "id": 2053, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pluginVersion": "7.1.5", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"})\n - sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"})\n ) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) * 100 ", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "A" + }, + { + "expr": "((sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (node) - sum(node_filesystem_free_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (node)) / sum(node_filesystem_size_bytes{device!~\"rootfs|HarddiskVolume.+\"}) by (node)) * 100\n", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ node }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Filesystem usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "folderId": 0, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 18 + }, + "headings": true, + "id": 3, + "limit": 30, + "links": [], + "query": "", + "recent": true, + "search": true, + "starred": false, + "tags": [], + "title": "Dashboards", + "type": "dashlist" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 2055, + "options": { + "content": "## About Rancher Monitoring\n\nRancher Monitoring is a Helm chart developed by Rancher that is powered by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). It is based on the upstream [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) Helm chart maintained by the Prometheus community.\n\nBy default, the chart deploys Grafana alongside a set of Grafana dashboards curated by the [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus) project.\n\nFor more information on how Rancher Monitoring differs from [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack), please view the CHANGELOG.md of the rancher-monitoring chart located in the [rancher/charts](https://github.com/rancher/charts) repository.\n\nFor more information about how to configure Rancher Monitoring, please view the [Rancher docs](https://rancher.com/docs/rancher/v2.x/en/).\n\n", + "mode": "markdown" + }, + "pluginVersion": "7.1.0", + "timeFrom": null, + "timeShift": null, + "title": "", + "type": "text" + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "hidden": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Home", + "uid": "hjvnWYFMz", + "version": 5 +} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/requirements.lock b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/requirements.lock new file mode 100755 index 000000000..9e0958f80 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/requirements.lock @@ -0,0 +1,54 @@ +dependencies: +- name: kube-state-metrics + repository: https://charts.helm.sh/stable/ + version: 2.8.14 +- name: prometheus-node-exporter + repository: https://prometheus-community.github.io/helm-charts + version: 1.11.2 +- name: grafana + repository: https://grafana.github.io/helm-charts + version: 5.6.4 +- name: prometheus-adapter + repository: https://prometheus-community.github.io/helm-charts + version: 2.7.1 +- name: rancher-pushprox + repository: file://../../rancher-pushprox/charts + version: 0.1.2 +- name: rancher-pushprox + repository: file://../../rancher-pushprox/charts + version: 0.1.2 +- name: rancher-pushprox + repository: file://../../rancher-pushprox/charts + version: 0.1.2 +- name: rancher-pushprox + repository: file://../../rancher-pushprox/charts + version: 0.1.2 +- name: rancher-pushprox + repository: file://../../rancher-pushprox/charts + version: 0.1.2 +- name: rancher-pushprox + repository: file://../../rancher-pushprox/charts + version: 0.1.2 +- name: rancher-pushprox + repository: file://../../rancher-pushprox/charts + version: 0.1.2 +- name: rancher-pushprox + repository: file://../../rancher-pushprox/charts + version: 0.1.2 +- name: rancher-pushprox + repository: file://../../rancher-pushprox/charts + version: 0.1.2 +- name: rancher-pushprox + repository: file://../../rancher-pushprox/charts + version: 0.1.2 +- name: rancher-pushprox + repository: file://../../rancher-pushprox/charts + version: 0.1.2 +- name: rancher-pushprox + repository: file://../../rancher-pushprox/charts + version: 0.1.2 +- name: rancher-pushprox + repository: file://../../rancher-pushprox/charts + version: 0.1.2 +digest: sha256:e9fe12d245bde664b9f3e332cd3a4c94e691e8a81120afa4631eb0041149e252 +generated: "2021-01-15T09:11:01.329739-08:00" diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/requirements.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/requirements.yaml new file mode 100755 index 000000000..ee75deffe --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/requirements.yaml @@ -0,0 +1,145 @@ +dependencies: +- name: grafana + version: "" + repository: file://./charts/grafana + condition: grafana.enabled + tags: [] + enabled: false + importvalues: [] + alias: "" +- name: k3sServer + version: "" + repository: file://./charts/k3sServer + condition: k3sServer.enabled + tags: [] + enabled: false + importvalues: [] + alias: "" +- name: kube-state-metrics + version: "" + repository: file://./charts/kube-state-metrics + condition: kubeStateMetrics.enabled + tags: [] + enabled: false + importvalues: [] + alias: "" +- name: kubeAdmControllerManager + version: "" + repository: file://./charts/kubeAdmControllerManager + condition: kubeAdmControllerManager.enabled + tags: [] + enabled: false + importvalues: [] + alias: "" +- name: kubeAdmEtcd + version: "" + repository: file://./charts/kubeAdmEtcd + condition: kubeAdmEtcd.enabled + tags: [] + enabled: false + importvalues: [] + alias: "" +- name: kubeAdmProxy + version: "" + repository: file://./charts/kubeAdmProxy + condition: kubeAdmProxy.enabled + tags: [] + enabled: false + importvalues: [] + alias: "" +- name: kubeAdmScheduler + version: "" + repository: file://./charts/kubeAdmScheduler + condition: kubeAdmScheduler.enabled + tags: [] + enabled: false + importvalues: [] + alias: "" +- name: prometheus-adapter + version: "" + repository: file://./charts/prometheus-adapter + condition: prometheus-adapter.enabled + tags: [] + enabled: false + importvalues: [] + alias: "" +- name: prometheus-node-exporter + version: "" + repository: file://./charts/prometheus-node-exporter + condition: nodeExporter.enabled + tags: [] + enabled: false + importvalues: [] + alias: "" +- name: rke2ControllerManager + version: "" + repository: file://./charts/rke2ControllerManager + condition: rke2ControllerManager.enabled + tags: [] + enabled: false + importvalues: [] + alias: "" +- name: rke2Etcd + version: "" + repository: file://./charts/rke2Etcd + condition: rke2Etcd.enabled + tags: [] + enabled: false + importvalues: [] + alias: "" +- name: rke2Proxy + version: "" + repository: file://./charts/rke2Proxy + condition: rke2Proxy.enabled + tags: [] + enabled: false + importvalues: [] + alias: "" +- name: rke2Scheduler + version: "" + repository: file://./charts/rke2Scheduler + condition: rke2Scheduler.enabled + tags: [] + enabled: false + importvalues: [] + alias: "" +- name: rkeControllerManager + version: "" + repository: file://./charts/rkeControllerManager + condition: rkeControllerManager.enabled + tags: [] + enabled: false + importvalues: [] + alias: "" +- name: rkeEtcd + version: "" + repository: file://./charts/rkeEtcd + condition: rkeEtcd.enabled + tags: [] + enabled: false + importvalues: [] + alias: "" +- name: rkeProxy + version: "" + repository: file://./charts/rkeProxy + condition: rkeProxy.enabled + tags: [] + enabled: false + importvalues: [] + alias: "" +- name: rkeScheduler + version: "" + repository: file://./charts/rkeScheduler + condition: rkeScheduler.enabled + tags: [] + enabled: false + importvalues: [] + alias: "" +- name: windowsExporter + version: "" + repository: file://./charts/windowsExporter + condition: windowsExporter.enabled + tags: [] + enabled: false + importvalues: [] + alias: "" diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/NOTES.txt b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/NOTES.txt new file mode 100755 index 000000000..371f3ae39 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/NOTES.txt @@ -0,0 +1,4 @@ +{{ $.Chart.Name }} has been installed. Check its status by running: + kubectl --namespace {{ template "kube-prometheus-stack.namespace" . }} get pods -l "release={{ $.Release.Name }}" + +Visit https://github.com/prometheus-operator/kube-prometheus for instructions on how to create & configure Alertmanager and Prometheus instances using the Operator. diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/_helpers.tpl b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/_helpers.tpl new file mode 100755 index 000000000..123cbad6d --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/_helpers.tpl @@ -0,0 +1,200 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +{{/* +https://github.com/helm/helm/issues/4535#issuecomment-477778391 +Usage: {{ include "call-nested" (list . "SUBCHART_NAME" "TEMPLATE") }} +e.g. {{ include "call-nested" (list . "grafana" "grafana.fullname") }} +*/}} +{{- define "call-nested" }} +{{- $dot := index . 0 }} +{{- $subchart := index . 1 | splitList "." }} +{{- $template := index . 2 }} +{{- $values := $dot.Values }} +{{- range $subchart }} +{{- $values = index $values . }} +{{- end }} +{{- include $template (dict "Chart" (dict "Name" (last $subchart)) "Values" $values "Release" $dot.Release "Capabilities" $dot.Capabilities) }} +{{- end }} + +# Special Exporters +{{- define "exporter.kubeEtcd.enabled" -}} +{{- if or .Values.kubeEtcd.enabled .Values.rkeEtcd.enabled .Values.kubeAdmEtcd.enabled .Values.rke2Etcd.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.enabled" -}} +{{- if or .Values.kubeControllerManager.enabled .Values.rkeControllerManager.enabled .Values.k3sServer.enabled .Values.kubeAdmControllerManager.enabled .Values.rke2ControllerManager.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.enabled" -}} +{{- if or .Values.kubeScheduler.enabled .Values.rkeScheduler.enabled .Values.k3sServer.enabled .Values.kubeAdmScheduler.enabled .Values.rke2Scheduler.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.enabled" -}} +{{- if or .Values.kubeProxy.enabled .Values.rkeProxy.enabled .Values.k3sServer.enabled .Values.kubeAdmProxy.enabled .Values.rke2Proxy.enabled -}} +"true" +{{- end -}} +{{- end }} + +{{- define "exporter.kubeControllerManager.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-controller-manager +{{- end -}} +{{- end }} + +{{- define "exporter.kubeScheduler.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-scheduler +{{- end -}} +{{- end }} + +{{- define "exporter.kubeProxy.jobName" -}} +{{- if .Values.k3sServer.enabled -}} +k3s-server +{{- else -}} +kube-proxy +{{- end -}} +{{- end }} + +{{- define "kubelet.serviceMonitor.resourcePath" -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if not (eq .Values.kubelet.serviceMonitor.resourcePath "/metrics/resource/v1alpha1") -}} +{{ .Values.kubelet.serviceMonitor.resourcePath }} +{{- else if semverCompare ">=1.20.0-0" $kubeTargetVersion -}} +/metrics/resource +{{- else -}} +/metrics/resource/v1alpha1 +{{- end -}} +{{- end }} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# Prometheus Operator + +{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. This is suffixed with -alertmanager, which means subtract 13 from longest 63 available */}} +{{- define "kube-prometheus-stack.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 50 | trimSuffix "-" -}} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "kube-prometheus-stack.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 26 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Fullname suffixed with operator */}} +{{- define "kube-prometheus-stack.operator.fullname" -}} +{{- printf "%s-operator" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} + +{{/* Fullname suffixed with prometheus */}} +{{- define "kube-prometheus-stack.prometheus.fullname" -}} +{{- printf "%s-prometheus" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} + +{{/* Fullname suffixed with alertmanager */}} +{{- define "kube-prometheus-stack.alertmanager.fullname" -}} +{{- printf "%s-alertmanager" (include "kube-prometheus-stack.fullname" .) -}} +{{- end }} + +{{/* Create chart name and version as used by the chart label. */}} +{{- define "kube-prometheus-stack.chartref" -}} +{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} +{{- end }} + +{{/* Generate basic labels */}} +{{- define "kube-prometheus-stack.labels" }} +chart: {{ template "kube-prometheus-stack.chartref" . }} +release: {{ $.Release.Name | quote }} +heritage: {{ $.Release.Service | quote }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end }} + +{{/* Create the name of kube-prometheus-stack service account to use */}} +{{- define "kube-prometheus-stack.operator.serviceAccountName" -}} +{{- if .Values.prometheusOperator.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.operator.fullname" .) .Values.prometheusOperator.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheusOperator.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of prometheus service account to use */}} +{{- define "kube-prometheus-stack.prometheus.serviceAccountName" -}} +{{- if .Values.prometheus.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.prometheus.fullname" .) .Values.prometheus.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.prometheus.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* Create the name of alertmanager service account to use */}} +{{- define "kube-prometheus-stack.alertmanager.serviceAccountName" -}} +{{- if .Values.alertmanager.serviceAccount.create -}} + {{ default (include "kube-prometheus-stack.alertmanager.fullname" .) .Values.alertmanager.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.alertmanager.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kube-prometheus-stack.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/alertmanager.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/alertmanager.yaml new file mode 100755 index 000000000..0e5a9c939 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/alertmanager.yaml @@ -0,0 +1,118 @@ +{{- if .Values.alertmanager.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: Alertmanager +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: +{{- if .Values.alertmanager.alertmanagerSpec.image }} + baseImage: {{ template "system_default_registry" . }}{{ .Values.alertmanager.alertmanagerSpec.image.repository }} + version: {{ .Values.alertmanager.alertmanagerSpec.image.tag }} + {{- if .Values.alertmanager.alertmanagerSpec.image.sha }} + sha: {{ .Values.alertmanager.alertmanagerSpec.image.sha }} + {{- end }} +{{- end }} + replicas: {{ .Values.alertmanager.alertmanagerSpec.replicas }} + listenLocal: {{ .Values.alertmanager.alertmanagerSpec.listenLocal }} + serviceAccountName: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} +{{- if .Values.alertmanager.alertmanagerSpec.externalUrl }} + externalUrl: "{{ .Values.alertmanager.alertmanagerSpec.externalUrl }}" +{{- else if and .Values.alertmanager.ingress.enabled .Values.alertmanager.ingress.hosts }} + externalUrl: "http://{{ index .Values.alertmanager.ingress.hosts 0 }}{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-alertmanager.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.alertmanager.service.port }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.nodeSelector }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.alertmanager.alertmanagerSpec.paused }} + logFormat: {{ .Values.alertmanager.alertmanagerSpec.logFormat | quote }} + logLevel: {{ .Values.alertmanager.alertmanagerSpec.logLevel | quote }} + retention: {{ .Values.alertmanager.alertmanagerSpec.retention | quote }} +{{- if .Values.alertmanager.alertmanagerSpec.secrets }} + secrets: +{{ toYaml .Values.alertmanager.alertmanagerSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configSecret }} + configSecret: {{ .Values.alertmanager.alertmanagerSpec.configSecret }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.configMaps }} + configMaps: +{{ toYaml .Values.alertmanager.alertmanagerSpec.configMaps | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.resources }} + resources: +{{ toYaml .Values.alertmanager.alertmanagerSpec.resources | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + routePrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.securityContext }} + securityContext: +{{ toYaml .Values.alertmanager.alertmanagerSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.storage }} + storage: +{{ toYaml .Values.alertmanager.alertmanagerSpec.storage | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.alertmanager.alertmanagerSpec.podMetadata | indent 4 }} +{{- end }} +{{- if or .Values.alertmanager.alertmanagerSpec.podAntiAffinity .Values.alertmanager.alertmanagerSpec.affinity }} + affinity: +{{- if .Values.alertmanager.alertmanagerSpec.affinity }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchLabels: + app: alertmanager + alertmanager: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- else if eq .Values.alertmanager.alertmanagerSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchLabels: + app: alertmanager + alertmanager: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- end }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.alertmanager.alertmanagerSpec.tolerations }} +{{ toYaml .Values.alertmanager.alertmanagerSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.containers }} + containers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.containers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.priorityClassName }} + priorityClassName: {{.Values.alertmanager.alertmanagerSpec.priorityClassName }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.additionalPeers }} + additionalPeers: +{{ toYaml .Values.alertmanager.alertmanagerSpec.additionalPeers | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumes }} + volumes: {{.Values.alertmanager.alertmanagerSpec.volumes }} +{{- end }} +{{- if .Values.alertmanager.alertmanagerSpec.volumeMounts }} + volumeMounts: {{.Values.alertmanager.alertmanagerSpec.volumeMounts }} +{{- end }} + portName: {{ .Values.alertmanager.alertmanagerSpec.portName }} +{{- end }} + diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/cleanupSecret.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/cleanupSecret.yaml new file mode 100755 index 000000000..908955697 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/cleanupSecret.yaml @@ -0,0 +1,88 @@ +{{- if and (.Values.alertmanager.enabled) (not .Values.alertmanager.alertmanagerSpec.useExistingSecret) (.Values.alertmanager.secret.cleanupOnUninstall) }} +apiVersion: batch/v1 +kind: Job +metadata: + name: alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-post-delete + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": hook-succeeded, hook-failed + "helm.sh/hook-weight": "5" +spec: + template: + metadata: + name: alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-post-delete + labels: {{ include "kube-prometheus-stack.labels" . | nindent 8 }} + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + spec: + serviceAccountName: alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-post-delete +{{- if .Values.alertmanager.secret.securityContext }} + securityContext: +{{ toYaml .Values.alertmanager.secret.securityContext | indent 8 }} +{{- end }} + containers: + - name: delete-secret + image: {{ template "system_default_registry" . }}{{ .Values.alertmanager.secret.image.repository }}:{{ .Values.alertmanager.secret.image.tag }} + imagePullPolicy: {{ .Values.alertmanager.secret.image.pullPolicy }} + command: + - /bin/sh + - -c + - > + if kubectl get secret -n {{ template "kube-prometheus-stack.namespace" . }} alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-alertmanager > /dev/null 2>&1; then + kubectl delete secret -n {{ template "kube-prometheus-stack.namespace" . }} alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-alertmanager + fi; + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-post-delete + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": hook-succeeded, hook-failed + "helm.sh/hook-weight": "3" +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: ['get', 'delete'] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-post-delete + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": hook-succeeded, hook-failed + "helm.sh/hook-weight": "3" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-post-delete +subjects: +- kind: ServiceAccount + name: alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-post-delete + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-post-delete + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": hook-succeeded, hook-failed + "helm.sh/hook-weight": "3" +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/ingress.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/ingress.yaml new file mode 100755 index 000000000..9c5f57048 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/ingress.yaml @@ -0,0 +1,53 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.ingress.enabled }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $servicePort := .Values.alertmanager.service.port -}} +{{- $routePrefix := list .Values.alertmanager.alertmanagerSpec.routePrefix }} +{{- $paths := .Values.alertmanager.ingress.paths | default $routePrefix -}} +{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} +apiVersion: networking.k8s.io/v1beta1 +{{ else }} +apiVersion: extensions/v1beta1 +{{ end -}} +kind: Ingress +metadata: + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.alertmanager.ingress.annotations }} + annotations: +{{ toYaml .Values.alertmanager.ingress.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.alertmanager.ingress.labels }} +{{ toYaml .Values.alertmanager.ingress.labels | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + rules: + {{- if .Values.alertmanager.ingress.hosts }} + {{- range $host := .Values.alertmanager.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + backend: + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + backend: + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end -}} + {{- end -}} + {{- if .Values.alertmanager.ingress.tls }} + tls: +{{ toYaml .Values.alertmanager.ingress.tls | indent 4 }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/ingressperreplica.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/ingressperreplica.yaml new file mode 100755 index 000000000..8342b2a8c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/ingressperreplica.yaml @@ -0,0 +1,53 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled .Values.alertmanager.ingressPerReplica.enabled }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $servicePort := .Values.alertmanager.service.port -}} +{{- $ingressValues := .Values.alertmanager.ingressPerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} + apiVersion: networking.k8s.io/v1beta1 + {{ else }} + apiVersion: extensions/v1beta1 + {{ end -}} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} + {{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: +{{ toYaml $ingressValues.annotations | indent 8 }} + {{- end }} + spec: + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + backend: + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + servicePort: {{ $servicePort }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/podDisruptionBudget.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/podDisruptionBudget.yaml new file mode 100755 index 000000000..1dbe809cd --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.podDisruptionBudget.enabled }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.alertmanager.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.alertmanager.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.alertmanager.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app: alertmanager + alertmanager: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/psp-role.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/psp-role.yaml new file mode 100755 index 000000000..d64d1f813 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/psp-role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.alertmanager.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/psp-rolebinding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/psp-rolebinding.yaml new file mode 100755 index 000000000..9248cc8dd --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/psp-rolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.alertmanager.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/psp.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/psp.yaml new file mode 100755 index 000000000..6fa445009 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/psp.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.alertmanager.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Required to prevent escalations to root. + # allowPrivilegeEscalation: false + # This is redundant with non-root + disallow privilege escalation, + # but we can provide it for defense in depth. + #requiredDropCapabilities: + # - ALL + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} + diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/secret.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/secret.yaml new file mode 100755 index 000000000..9d9bdabfa --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/secret.yaml @@ -0,0 +1,166 @@ +{{- if and (.Values.alertmanager.enabled) (not .Values.alertmanager.alertmanagerSpec.useExistingSecret) }} +{{- if .Release.IsInstall }} +{{- $secretName := (printf "alertmanager-%s-alertmanager" (include "kube-prometheus-stack.fullname" .)) }} +{{- if (lookup "v1" "Secret" (include "kube-prometheus-stack.namespace" .) $secretName) }} +{{- required (printf "Cannot overwrite existing secret %s in namespace %s." $secretName (include "kube-prometheus-stack.namespace" .)) "" }} +{{- end }}{{- end }} +apiVersion: v1 +kind: Secret +metadata: + name: alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-pre-install + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install + "helm.sh/hook-delete-policy": hook-succeeded, hook-failed + "helm.sh/hook-weight": "3" +{{- if .Values.alertmanager.secret.annotations }} +{{ toYaml .Values.alertmanager.secret.annotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: +{{- if .Values.alertmanager.tplConfig }} + alertmanager.yaml: {{ tpl (toYaml .Values.alertmanager.config) . | b64enc | quote }} +{{- else }} + alertmanager.yaml: {{ toYaml .Values.alertmanager.config | b64enc | quote }} +{{- end}} +{{- range $key, $val := .Values.alertmanager.templateFiles }} + {{ $key }}: {{ $val | b64enc | quote }} +{{- end }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-pre-install + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + annotations: + "helm.sh/hook": pre-install + "helm.sh/hook-delete-policy": hook-succeeded, hook-failed + "helm.sh/hook-weight": "5" +spec: + template: + metadata: + name: alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-pre-install + labels: {{ include "kube-prometheus-stack.labels" . | nindent 8 }} + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + spec: + serviceAccountName: alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-pre-install +{{- if .Values.alertmanager.secret.securityContext }} + securityContext: +{{ toYaml .Values.alertmanager.secret.securityContext | indent 8 }} +{{- end }} + containers: + - name: copy-pre-install-secret + image: {{ template "system_default_registry" . }}{{ .Values.alertmanager.secret.image.repository }}:{{ .Values.alertmanager.secret.image.tag }} + imagePullPolicy: {{ .Values.alertmanager.secret.image.pullPolicy }} + command: + - /bin/sh + - -c + - > + if kubectl get secret -n {{ template "kube-prometheus-stack.namespace" . }} alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-alertmanager > /dev/null 2>&1; then + echo "Secret already exists" + exit 1 + fi; + kubectl patch secret -n {{ template "kube-prometheus-stack.namespace" . }} --dry-run -o yaml + alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-pre-install + -p '{{ printf "{\"metadata\":{\"name\": \"alertmanager-%s-alertmanager\"}}" (include "kube-prometheus-stack.fullname" .) }}' + | kubectl apply -f -; + kubectl annotate secret -n {{ template "kube-prometheus-stack.namespace" . }} + alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-alertmanager + helm.sh/hook- helm.sh/hook-delete-policy- helm.sh/hook-weight-; + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-pre-install + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + annotations: + "helm.sh/hook": pre-install + "helm.sh/hook-delete-policy": hook-succeeded, hook-failed + "helm.sh/hook-weight": "3" +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: ['create', 'get', 'patch'] +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-pre-install +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-pre-install + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + annotations: + "helm.sh/hook": pre-install + "helm.sh/hook-delete-policy": hook-succeeded, hook-failed + "helm.sh/hook-weight": "3" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-pre-install +subjects: +- kind: ServiceAccount + name: alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-pre-install + namespace: {{ template "kube-prometheus-stack.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-pre-install + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + annotations: + "helm.sh/hook": pre-install + "helm.sh/hook-delete-policy": hook-succeeded, hook-failed + "helm.sh/hook-weight": "3" +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: alertmanager-{{ template "kube-prometheus-stack.fullname" . }}-pre-install + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + annotations: + "helm.sh/hook": pre-install + "helm.sh/hook-delete-policy": hook-succeeded, hook-failed + "helm.sh/hook-weight": "3" +spec: + privileged: false + allowPrivilegeEscalation: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/service.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/service.yaml new file mode 100755 index 000000000..f1a63c2e3 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/service.yaml @@ -0,0 +1,47 @@ +{{- if .Values.alertmanager.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + self-monitor: {{ .Values.alertmanager.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.service.labels }} +{{ toYaml .Values.alertmanager.service.labels | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.annotations }} + annotations: +{{ toYaml .Values.alertmanager.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.alertmanager.service.clusterIP }} + clusterIP: {{ .Values.alertmanager.service.clusterIP }} +{{- end }} +{{- if .Values.alertmanager.service.externalIPs }} + externalIPs: +{{ toYaml .Values.alertmanager.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.alertmanager.service.loadBalancerIP }} +{{- end }} +{{- if .Values.alertmanager.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.alertmanager.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} + ports: + - name: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq .Values.alertmanager.service.type "NodePort" }} + nodePort: {{ .Values.alertmanager.service.nodePort }} + {{- end }} + port: {{ .Values.alertmanager.service.port }} + targetPort: {{ .Values.alertmanager.service.targetPort }} + protocol: TCP + selector: + app: alertmanager + alertmanager: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + type: "{{ .Values.alertmanager.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/serviceaccount.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/serviceaccount.yaml new file mode 100755 index 000000000..c5e6e9228 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.alertmanager.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.alertmanager.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.alertmanager.serviceAccount.annotations | indent 4 }} +{{- end }} +imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/servicemonitor.yaml new file mode 100755 index 000000000..b9bc14fe6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/servicemonitor.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-alertmanager + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if .Values.alertmanager.serviceMonitor.interval }} + interval: {{ .Values.alertmanager.serviceMonitor.interval }} + {{- end }} + path: "{{ trimSuffix "/" .Values.alertmanager.alertmanagerSpec.routePrefix }}/metrics" +{{- if .Values.alertmanager.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.alertmanager.serviceMonitor.metricRelabelings | indent 6) . }} +{{- end }} +{{- if .Values.alertmanager.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.alertmanager.serviceMonitor.relabelings | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/serviceperreplica.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/serviceperreplica.yaml new file mode 100755 index 000000000..0f12ae879 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/alertmanager/serviceperreplica.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.alertmanager.enabled .Values.alertmanager.servicePerReplica.enabled }} +{{- $count := .Values.alertmanager.alertmanagerSpec.replicas | int -}} +{{- $serviceValues := .Values.alertmanager.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-alertmanager +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + ports: + - name: {{ $.Values.alertmanager.alertmanagerSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + app: alertmanager + alertmanager: {{ template "kube-prometheus-stack.fullname" $ }}-alertmanager + statefulset.kubernetes.io/pod-name: alertmanager-{{ include "kube-prometheus-stack.fullname" $ }}-alertmanager-{{ $i }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/core-dns/service.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/core-dns/service.yaml new file mode 100755 index 000000000..f77db4199 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/core-dns/service.yaml @@ -0,0 +1,24 @@ +{{- if .Values.coreDns.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + jobLabel: coredns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.coreDns.service.port }} + protocol: TCP + targetPort: {{ .Values.coreDns.service.targetPort }} + selector: + {{- if .Values.coreDns.service.selector }} +{{ toYaml .Values.coreDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/core-dns/servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/core-dns/servicemonitor.yaml new file mode 100755 index 000000000..f34549048 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/core-dns/servicemonitor.yaml @@ -0,0 +1,33 @@ +{{- if .Values.coreDns.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-coredns + namespace: "kube-system" + labels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-coredns + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics + {{- if .Values.coreDns.serviceMonitor.interval}} + interval: {{ .Values.coreDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.coreDns.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.coreDns.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.coreDns.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.coreDns.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-api-server/servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-api-server/servicemonitor.yaml new file mode 100755 index 000000000..b7ea3817c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-api-server/servicemonitor.yaml @@ -0,0 +1,36 @@ +{{- if .Values.kubeApiServer.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-apiserver + namespace: default + labels: + app: {{ template "kube-prometheus-stack.name" . }}-apiserver +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeApiServer.serviceMonitor.interval }} + interval: {{ .Values.kubeApiServer.serviceMonitor.interval }} + {{- end }} + port: https + scheme: https +{{- if .Values.kubeApiServer.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubeApiServer.serviceMonitor.metricRelabelings | indent 6) . }} +{{- end }} +{{- if .Values.kubeApiServer.relabelings }} + relabelings: +{{ toYaml .Values.kubeApiServer.relabelings | indent 6 }} +{{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + serverName: {{ .Values.kubeApiServer.tlsConfig.serverName }} + insecureSkipVerify: {{ .Values.kubeApiServer.tlsConfig.insecureSkipVerify }} + jobLabel: {{ .Values.kubeApiServer.serviceMonitor.jobLabel }} + namespaceSelector: + matchNames: + - default + selector: +{{ toYaml .Values.kubeApiServer.serviceMonitor.selector | indent 4 }} +{{- end}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-controller-manager/endpoints.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-controller-manager/endpoints.yaml new file mode 100755 index 000000000..413193028 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-controller-manager/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeControllerManager.enabled .Values.kubeControllerManager.endpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + k8s-app: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeControllerManager.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: http-metrics + port: {{ .Values.kubeControllerManager.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-controller-manager/service.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-controller-manager/service.yaml new file mode 100755 index 000000000..8b5556a89 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-controller-manager/service.yaml @@ -0,0 +1,27 @@ +{{- if .Values.kubeControllerManager.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + jobLabel: kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.kubeControllerManager.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeControllerManager.service.targetPort }} +{{- if .Values.kubeControllerManager.endpoints }}{{- else }} + selector: + {{- if .Values.kubeControllerManager.service.selector }} +{{ toYaml .Values.kubeControllerManager.service.selector | indent 4 }} + {{- else}} + component: kube-controller-manager + {{- end}} +{{- end }} + type: ClusterIP +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-controller-manager/servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-controller-manager/servicemonitor.yaml new file mode 100755 index 000000000..4b762cd98 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-controller-manager/servicemonitor.yaml @@ -0,0 +1,44 @@ +{{- if .Values.kubeControllerManager.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-controller-manager + namespace: "kube-system" + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-controller-manager + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics + {{- if .Values.kubeControllerManager.serviceMonitor.interval }} + interval: {{ .Values.kubeControllerManager.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeControllerManager.serviceMonitor.https }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if .Values.kubeControllerManager.serviceMonitor.insecureSkipVerify }} + insecureSkipVerify: {{ .Values.kubeControllerManager.serviceMonitor.insecureSkipVerify }} + {{- end }} + {{- if .Values.kubeControllerManager.serviceMonitor.serverName }} + serverName: {{ .Values.kubeControllerManager.serviceMonitor.serverName }} + {{- end }} + {{- end }} +{{- if .Values.kubeControllerManager.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubeControllerManager.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubeControllerManager.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.kubeControllerManager.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-dns/service.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-dns/service.yaml new file mode 100755 index 000000000..c7bf142d5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-dns/service.yaml @@ -0,0 +1,28 @@ +{{- if .Values.kubeDns.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + jobLabel: kube-dns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics-dnsmasq + port: {{ .Values.kubeDns.service.dnsmasq.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.dnsmasq.targetPort }} + - name: http-metrics-skydns + port: {{ .Values.kubeDns.service.skydns.port }} + protocol: TCP + targetPort: {{ .Values.kubeDns.service.skydns.targetPort }} + selector: + {{- if .Values.kubeDns.service.selector }} +{{ toYaml .Values.kubeDns.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-dns + {{- end}} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-dns/servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-dns/servicemonitor.yaml new file mode 100755 index 000000000..28d06ae83 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-dns/servicemonitor.yaml @@ -0,0 +1,46 @@ +{{- if .Values.kubeDns.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-dns + namespace: "kube-system" + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-dns + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics-dnsmasq + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.dnsmasqMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubeDns.serviceMonitor.dnsmasqRelabelings }} + relabelings: +{{ toYaml .Values.kubeDns.serviceMonitor.dnsmasqRelabelings | indent 4 }} +{{- end }} + - port: http-metrics-skydns + {{- if .Values.kubeDns.serviceMonitor.interval }} + interval: {{ .Values.kubeDns.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubeDns.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubeDns.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubeDns.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.kubeDns.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-etcd/endpoints.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-etcd/endpoints.yaml new file mode 100755 index 000000000..8f07a5cc2 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-etcd/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeEtcd.enabled .Values.kubeEtcd.endpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + k8s-app: etcd-server +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeEtcd.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: http-metrics + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-etcd/service.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-etcd/service.yaml new file mode 100755 index 000000000..957e8bbc1 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-etcd/service.yaml @@ -0,0 +1,27 @@ +{{- if .Values.kubeEtcd.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + jobLabel: kube-etcd +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.kubeEtcd.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeEtcd.service.targetPort }} +{{- if .Values.kubeEtcd.endpoints }}{{- else }} + selector: + {{- if .Values.kubeEtcd.service.selector }} +{{ toYaml .Values.kubeEtcd.service.selector | indent 4 }} + {{- else}} + component: etcd + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-etcd/servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-etcd/servicemonitor.yaml new file mode 100755 index 000000000..17a447ad9 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-etcd/servicemonitor.yaml @@ -0,0 +1,50 @@ +{{- if .Values.kubeEtcd.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-etcd + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-etcd + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics + {{- if .Values.kubeEtcd.serviceMonitor.interval }} + interval: {{ .Values.kubeEtcd.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if eq .Values.kubeEtcd.serviceMonitor.scheme "https" }} + scheme: https + tlsConfig: + {{- if .Values.kubeEtcd.serviceMonitor.serverName }} + serverName: {{ .Values.kubeEtcd.serviceMonitor.serverName }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.caFile }} + caFile: {{ .Values.kubeEtcd.serviceMonitor.caFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.certFile }} + certFile: {{ .Values.kubeEtcd.serviceMonitor.certFile }} + {{- end }} + {{- if .Values.kubeEtcd.serviceMonitor.keyFile }} + keyFile: {{ .Values.kubeEtcd.serviceMonitor.keyFile }} + {{- end}} + insecureSkipVerify: {{ .Values.kubeEtcd.serviceMonitor.insecureSkipVerify }} + {{- end }} +{{- if .Values.kubeEtcd.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubeEtcd.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubeEtcd.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.kubeEtcd.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-proxy/endpoints.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-proxy/endpoints.yaml new file mode 100755 index 000000000..2cb756d15 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-proxy/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeProxy.enabled .Values.kubeProxy.endpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + k8s-app: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeProxy.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: http-metrics + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-proxy/service.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-proxy/service.yaml new file mode 100755 index 000000000..abe420600 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-proxy/service.yaml @@ -0,0 +1,27 @@ +{{- if .Values.kubeProxy.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + jobLabel: kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.kubeProxy.service.port }} + protocol: TCP + targetPort: {{ .Values.kubeProxy.service.targetPort }} +{{- if .Values.kubeProxy.endpoints }}{{- else }} + selector: + {{- if .Values.kubeProxy.service.selector }} +{{ toYaml .Values.kubeProxy.service.selector | indent 4 }} + {{- else}} + k8s-app: kube-proxy + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-proxy/servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-proxy/servicemonitor.yaml new file mode 100755 index 000000000..0ac44182b --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-proxy/servicemonitor.yaml @@ -0,0 +1,38 @@ +{{- if .Values.kubeProxy.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-proxy + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-proxy + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics + {{- if .Values.kubeProxy.serviceMonitor.interval }} + interval: {{ .Values.kubeProxy.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeProxy.serviceMonitor.https }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- end}} +{{- if .Values.kubeProxy.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ toYaml .Values.kubeProxy.serviceMonitor.metricRelabelings | indent 4 }} +{{- end }} +{{- if .Values.kubeProxy.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.kubeProxy.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-scheduler/endpoints.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-scheduler/endpoints.yaml new file mode 100755 index 000000000..f4ad60fd6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-scheduler/endpoints.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.kubeScheduler.enabled .Values.kubeScheduler.endpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + k8s-app: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +subsets: + - addresses: + {{- range .Values.kubeScheduler.endpoints }} + - ip: {{ . }} + {{- end }} + ports: + - name: http-metrics + port: {{ .Values.kubeScheduler.service.port }} + protocol: TCP +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-scheduler/service.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-scheduler/service.yaml new file mode 100755 index 000000000..15559d45a --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-scheduler/service.yaml @@ -0,0 +1,27 @@ +{{- if .Values.kubeScheduler.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + jobLabel: kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: kube-system +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.kubeScheduler.service.port}} + protocol: TCP + targetPort: {{ .Values.kubeScheduler.service.targetPort}} +{{- if .Values.kubeScheduler.endpoints }}{{- else }} + selector: + {{- if .Values.kubeScheduler.service.selector }} +{{ toYaml .Values.kubeScheduler.service.selector | indent 4 }} + {{- else}} + component: kube-scheduler + {{- end}} +{{- end }} + type: ClusterIP +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-scheduler/servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-scheduler/servicemonitor.yaml new file mode 100755 index 000000000..b565ec2b8 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-scheduler/servicemonitor.yaml @@ -0,0 +1,44 @@ +{{- if .Values.kubeScheduler.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-scheduler + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-scheduler + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - "kube-system" + endpoints: + - port: http-metrics + {{- if .Values.kubeScheduler.serviceMonitor.interval }} + interval: {{ .Values.kubeScheduler.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + {{- if .Values.kubeScheduler.serviceMonitor.https }} + scheme: https + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + {{- if .Values.kubeScheduler.serviceMonitor.insecureSkipVerify }} + insecureSkipVerify: {{ .Values.kubeScheduler.serviceMonitor.insecureSkipVerify }} + {{- end}} + {{- if .Values.kubeScheduler.serviceMonitor.serverName }} + serverName: {{ .Values.kubeScheduler.serviceMonitor.serverName }} + {{- end}} + {{- end}} +{{- if .Values.kubeScheduler.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubeScheduler.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubeScheduler.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.kubeScheduler.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-state-metrics/serviceMonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-state-metrics/serviceMonitor.yaml new file mode 100755 index 000000000..f09de5d2e --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kube-state-metrics/serviceMonitor.yaml @@ -0,0 +1,30 @@ +{{- if .Values.kubeStateMetrics.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kube-state-metrics + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kube-state-metrics +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: app.kubernetes.io/name + endpoints: + - port: http + {{- if .Values.kubeStateMetrics.serviceMonitor.interval }} + interval: {{ .Values.kubeStateMetrics.serviceMonitor.interval }} + {{- end }} + honorLabels: true +{{- if .Values.kubeStateMetrics.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubeStateMetrics.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubeStateMetrics.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.kubeStateMetrics.serviceMonitor.relabelings | indent 4 }} +{{- end }} + selector: + matchLabels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/instance: "{{ $.Release.Name }}" +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kubelet/servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kubelet/servicemonitor.yaml new file mode 100755 index 000000000..15811312c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/kubelet/servicemonitor.yaml @@ -0,0 +1,151 @@ +{{- if .Values.kubelet.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-kubelet + namespace: {{ .Values.kubelet.namespace }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-kubelet +{{- include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + endpoints: + {{- if .Values.kubelet.serviceMonitor.https }} + - port: https-metrics + scheme: https + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + honorLabels: true +{{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: https-metrics + scheme: https + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + {{- if .Values.kubelet.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.kubelet.serviceMonitor.scrapeTimeout }} + {{- end }} + honorLabels: true + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4 }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probes }} + - port: https-metrics + scheme: https + path: /metrics/probes + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + honorLabels: true + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.probesMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.probesMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.probesRelabelings }} + relabelings: +{{ toYaml .Values.kubelet.serviceMonitor.probesRelabelings | indent 4 }} +{{- end }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: https-metrics + scheme: https + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + honorLabels: true + tlsConfig: + caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecureSkipVerify: true + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4 }} +{{- end }} +{{- end }} + {{- else }} + - port: http-metrics + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + honorLabels: true +{{- if .Values.kubelet.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.kubelet.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisor }} + - port: http-metrics + path: /metrics/cadvisor + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + honorLabels: true +{{- if .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.cAdvisorMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.cAdvisorRelabelings }} + relabelings: +{{ toYaml .Values.kubelet.serviceMonitor.cAdvisorRelabelings | indent 4 }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resource }} + - port: http-metrics + path: {{ include "kubelet.serviceMonitor.resourcePath" . }} + {{- if .Values.kubelet.serviceMonitor.interval }} + interval: {{ .Values.kubelet.serviceMonitor.interval }} + {{- end }} + honorLabels: true +{{- if .Values.kubelet.serviceMonitor.resourceMetricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.kubelet.serviceMonitor.resourceMetricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.kubelet.serviceMonitor.resourceRelabelings }} + relabelings: +{{ toYaml .Values.kubelet.serviceMonitor.resourceRelabelings | indent 4 }} +{{- end }} +{{- end }} +{{- end }} + {{- end }} + jobLabel: k8s-app + namespaceSelector: + matchNames: + - {{ .Values.kubelet.namespace }} + selector: + matchLabels: + k8s-app: kubelet +{{- end}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/node-exporter/servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/node-exporter/servicemonitor.yaml new file mode 100755 index 000000000..5ca5f1b75 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/exporters/node-exporter/servicemonitor.yaml @@ -0,0 +1,32 @@ +{{- if .Values.nodeExporter.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-node-exporter + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-node-exporter +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: {{ .Values.nodeExporter.jobLabel }} + selector: + matchLabels: + app: prometheus-node-exporter + release: {{ $.Release.Name }} + endpoints: + - port: metrics + {{- if .Values.nodeExporter.serviceMonitor.interval }} + interval: {{ .Values.nodeExporter.serviceMonitor.interval }} + {{- end }} + {{- if .Values.nodeExporter.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.nodeExporter.serviceMonitor.scrapeTimeout }} + {{- end }} +{{- if .Values.nodeExporter.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.nodeExporter.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.nodeExporter.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.nodeExporter.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/configmap-dashboards.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/configmap-dashboards.yaml new file mode 100755 index 000000000..f11af8285 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/configmap-dashboards.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- $files := .Files.Glob "dashboards/*.json" }} +{{- if $files }} +apiVersion: v1 +kind: ConfigMapList +items: +{{- range $path, $fileContents := $files }} +{{- $dashboardName := regexReplaceAll "(^.*/)(.*)\\.json$" $path "${2}" }} +- apiVersion: v1 + kind: ConfigMap + metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) $dashboardName | trunc 63 | trimSuffix "-" }} + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 6 }} + data: + {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/configmaps-datasources.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/configmaps-datasources.yaml new file mode 100755 index 000000000..1e978907f --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/configmaps-datasources.yaml @@ -0,0 +1,38 @@ +{{- if and .Values.grafana.enabled .Values.grafana.sidecar.datasources.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-grafana-datasource + namespace: {{ default .Values.grafana.sidecar.datasources.searchNamespace (include "kube-prometheus-stack.namespace" .) }} +{{- if .Values.grafana.sidecar.datasources.annotations }} + annotations: +{{ toYaml .Values.grafana.sidecar.datasources.annotations | indent 4 }} +{{- end }} + labels: + {{ $.Values.grafana.sidecar.datasources.label }}: "1" + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + datasource.yaml: |- + apiVersion: 1 + datasources: +{{- if .Values.grafana.sidecar.datasources.defaultDatasourceEnabled }} + - name: Prometheus + type: prometheus + url: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}/{{ trimPrefix "/" .Values.prometheus.prometheusSpec.routePrefix }} + access: proxy + isDefault: true +{{- if .Values.grafana.sidecar.datasources.createPrometheusReplicasDatasources }} +{{- range until (int .Values.prometheus.prometheusSpec.replicas) }} + - name: Prometheus-{{ . }} + type: prometheus + url: http://prometheus-{{ template "kube-prometheus-stack.fullname" $ }}-prometheus-{{ . }}.prometheus-operated:9090/{{ trimPrefix "/" $.Values.prometheus.prometheusSpec.routePrefix }} + access: proxy + isDefault: false +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.grafana.additionalDataSources }} +{{ tpl (toYaml .Values.grafana.additionalDataSources | indent 4) . }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/apiserver.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/apiserver.yaml new file mode 100755 index 000000000..266eefa12 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/apiserver.yaml @@ -0,0 +1,1734 @@ +{{- /* +Generated from 'apiserver' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.kubeApiServer.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "apiserver" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + apiserver.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "content": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "datasource": null, + "description": "The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "mode": "markdown", + "span": 12, + "title": "Notice", + "type": "text" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of requests (both read and write) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Availability (30d) > 99.000%", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "decimals": 3, + "description": "How much error budget is left looking at our 0.990% availability gurantees?", + "fill": 10, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 * (apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"} - 0.990000)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "errorbudget", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "ErrorBudget (30d) > 99.000%", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 3, + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of read requests (LIST,GET) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Read Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many read requests (LIST,GET) per second do the apiservers get by code?", + "fill": 10, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of read requests (LIST,GET) per second are returned with errors (5xx)?", + "fill": 1, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for reading (LIST|GET) a given resource?", + "fill": 1, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{verb=\"read\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Read SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 3, + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) in 30 days have been answered successfully and fast enough?", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 9, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "apiserver_request:availability30d{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Write Availability (30d)", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many write requests (POST|PUT|PATCH|DELETE) per second do the apiservers get by code?", + "fill": 10, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/2../i", + "color": "#56A64B" + }, + { + "alias": "/3../i", + "color": "#F2CC0C" + }, + { + "alias": "/4../i", + "color": "#3274D9" + }, + { + "alias": "/5../i", + "color": "#E02F44" + } + ], + "spaceLength": 10, + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} code {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Requests", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "reqps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many percent of write requests (POST|PUT|PATCH|DELETE) per second are returned with errors (5xx)?", + "fill": 1, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Errors", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "How many seconds is the 99th percentile for writing (POST|PUT|PATCH|DELETE) a given resource?", + "fill": 1, + "gridPos": { + + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{verb=\"write\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}} resource {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Write SLI - Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[5m])) by (instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 17, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 18, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(apiserver_request_total, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(apiserver_request_total{job=\"apiserver\", cluster=\"$cluster\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Kubernetes / API server", + "uid": "09ec8aa1e996d6ffcd6817bbaff4db1b", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/cluster-total.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/cluster-total.yaml new file mode 100755 index 000000000..489a1a404 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/cluster-total.yaml @@ -0,0 +1,1841 @@ +{{- /* +Generated from 'cluster-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "cluster-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + cluster-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "namespace", + "value": "namespace" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/8b7a8b326d7a6f1f04244066368c67af/kubernetes-networking-namespace-pods?orgId=1&refresh=30s&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth History", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "What is TCP Retransmit?", + "url": "https://accedian.com/enterprises/blog/network-packet-loss-retransmissions-and-duplicate-acknowledgements/" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_Tcp_RetransSegs[$interval:$resolution]) / rate(node_netstat_Tcp_OutSegs[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP Retransmits out of all sent segments", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "Why monitor SYN retransmits?", + "url": "https://github.com/prometheus/node_exporter/issues/1023#issuecomment-408128365" + } + ], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(node_netstat_TcpExt_TCPSynRetrans[$interval:$resolution]) / rate(node_netstat_Tcp_RetransSegs[$interval:$resolution])) by (instance))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of TCP SYN Retransmits out of all retransmits", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Kubernetes / Networking / Cluster", + "uid": "ff635a025bcfea7bc3dd4f508990a3e9", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/controller-manager.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/controller-manager.yaml new file mode 100755 index 000000000..51a825c95 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/controller-manager.yaml @@ -0,0 +1,1144 @@ +{{- /* +Generated from 'controller-manager' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "controller-manager" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + controller-manager.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(up{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[5m])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[5m])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\"}[5m])) by (instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}name{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(process_cpu_seconds_total{job=\"{{ include "exporter.kubeControllerManager.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Kubernetes / Controller Manager", + "uid": "72e0e05bef5099e5f049b05fdc429ed4", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/etcd.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/etcd.yaml new file mode 100755 index 000000000..b81cda955 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/etcd.yaml @@ -0,0 +1,1118 @@ +{{- /* +Generated from 'etcd' from https://raw.githubusercontent.com/etcd-io/etcd/master/Documentation/op-guide/grafana.json +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "etcd" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + etcd.json: |- + { + "annotations": { + "list": [] + }, + "description": "etcd sample Grafana dashboard with Prometheus", + "editable": true, + "gnetId": null, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 28, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum(etcd_server_has_leader{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "", + "metric": "etcd_server_has_leader", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "title": "Up", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 23, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{job=\"$cluster\",grpc_type=\"unary\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Rate", + "metric": "grpc_server_started_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(rate(grpc_server_handled_total{job=\"$cluster\",grpc_type=\"unary\",grpc_code!=\"OK\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Failed Rate", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 41, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Watch Streams", + "metric": "grpc_server_handled_total", + "refId": "A", + "step": 4 + }, + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Lease Streams", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "showTitle": false, + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "etcd_mvcc_db_total_size_in_bytes{job=\"$cluster\"}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB Size", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "DB Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$cluster\"}[5m])) by (instance, le))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} WAL fsync", + "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$cluster\"}[5m])) by (instance, le))", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB fsync", + "metric": "etcd_disk_backend_commit_duration_seconds_bucket", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 29, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"$cluster\"}", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Resident Memory", + "metric": "process_resident_memory_bytes", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 22, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"$cluster\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic In", + "metric": "etcd_network_client_grpc_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 21, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"$cluster\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic Out", + "metric": "etcd_network_client_grpc_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 20, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"$cluster\"}[5m])) by (instance)", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic In", + "metric": "etcd_network_peer_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"$cluster\"}[5m])) by (instance)", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic Out", + "metric": "etcd_network_peer_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 40, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"$cluster\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "Proposal Failure Rate", + "metric": "etcd_server_proposals_failed_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(etcd_server_proposals_pending{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "Proposal Pending Total", + "metric": "etcd_server_proposals_pending", + "refId": "B", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"$cluster\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "Proposal Commit Rate", + "metric": "etcd_server_proposals_committed_total", + "refId": "C", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "Proposal Apply Rate", + "refId": "D", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fill": 0, + "id": 19, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "changes(etcd_server_leader_changes_seen_total{job=\"$cluster\"}[1d])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total Leader Elections Per Day", + "metric": "etcd_server_leader_changes_seen_total", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Leader Elections Per Day", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + } + ], + "schemaVersion": 13, + "sharedCrosshair": false, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(etcd_server_has_leader, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "etcd", + "uid": "c2f4e12cdf69feb95caa41a5a1b423d9", + "version": 215 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-coredns.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-coredns.yaml new file mode 100755 index 000000000..c79f96761 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-coredns.yaml @@ -0,0 +1,1340 @@ +{{- /* Added manually, can be changed in-place. */ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.coreDns.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-coredns" | trunc 63 | trimSuffix "-" }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-coredns.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m])) by (proto)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{proto}}`}}", + "refId": "A", + "step": 60 + }, + { + "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Requests (total)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + }, + { + "alias": "other", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_type_count_total{instance=~\"$instance\"}[5m])) by (type)", + "intervalFactor": 2, + "legendFormat": "{{`{{type}}`}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Requests (by qtype)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m])) by (zone)", + "intervalFactor": 2, + "legendFormat": "{{`{{zone}}`}}", + "refId": "A", + "step": 60 + }, + { + "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Requests (by zone)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_do_count_total{instance=~\"$instance\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "DO", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Requests (DO bit)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 7 + }, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90%", + "yaxis": 2 + }, + { + "alias": "tcp:99%", + "yaxis": 2 + }, + { + "alias": "tcp:50%", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{`{{proto}}`}}:99%", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{`{{proto}}`}}:90%", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{`{{proto}}`}}:50%", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Requests (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90%", + "yaxis": 1 + }, + { + "alias": "tcp:99%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{`{{proto}}`}}:99%", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{`{{proto}}`}}:90%", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{`{{proto}}`}}:50%", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Requests (size, tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_response_rcode_count_total{instance=~\"$instance\"}[5m])) by (rcode)", + "intervalFactor": 2, + "legendFormat": "{{`{{rcode}}`}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Responses (by rcode)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_duration_seconds_bucket{instance=~\"$instance\"}[5m])) by (le, job))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_duration_seconds_bucket{instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_duration_seconds_bucket{instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50%", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Responses (duration)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 2 + }, + { + "alias": "tcp:90%", + "yaxis": 2 + }, + { + "alias": "tcp:99%", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "intervalFactor": 2, + "legendFormat": "{{`{{proto}}`}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "intervalFactor": 2, + "legendFormat": "{{`{{proto}}`}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "intervalFactor": 2, + "legendFormat": "{{`{{proto}}`}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Responses (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:90%", + "yaxis": 1 + }, + { + "alias": "tcp:99%", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{proto}}`}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{proto}}`}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le, proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{proto}}`}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Responses (size, tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(coredns_cache_size{instance=~\"$instance\"}) by (type)", + "intervalFactor": 2, + "legendFormat": "{{`{{type}}`}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Cache (size)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "misses", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_cache_hits_total{instance=~\"$instance\"}[5m])) by (type)", + "intervalFactor": 2, + "legendFormat": "hits:{{`{{type}}`}}", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_cache_misses_total{instance=~\"$instance\"}[5m])) by (type)", + "intervalFactor": 2, + "legendFormat": "misses", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Cache (hitrate)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "tags": [], + "text": "172.16.1.8:9153", + "value": "172.16.1.8:9153" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "up{job=\"coredns\"}", + "refresh": 1, + "regex": ".*instance=\"(.*?)\".*", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "CoreDNS", + "uid": "vkQ0UHxik", + "version": 1 + } +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml new file mode 100755 index 000000000..1ed508ed3 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml @@ -0,0 +1,2582 @@ +{{- /* +Generated from 'k8s-resources-cluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-cluster.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"$cluster\"}[$__interval]))", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_cpu_cores{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Requests Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_cpu_cores{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Limits Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Requests Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Limits Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "./d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to workloads", + "linkUrl": "./d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(kube_pod_owner{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests by Namespace", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Requests", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 11, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Received", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Transmitted", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$__interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(node_cpu_seconds_total, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Kubernetes / Compute Resources / Cluster", + "uid": "efa86fd1d0c121a26444b636a3f509a8", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml new file mode 100755 index 000000000..983720d46 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml @@ -0,0 +1,2286 @@ +{{- /* +Generated from 'k8s-resources-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from requests)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilization (from requests)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation (from limits)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Kubernetes / Compute Resources / Namespace (Pods)", + "uid": "85a562078cdf77779eaa1add43ccec1e", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-node.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-node.yaml new file mode 100755 index 000000000..dc84a6b4c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-node.yaml @@ -0,0 +1,978 @@ +{{- /* +Generated from 'k8s-resources-node' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-node" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-node.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\", container!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{node=~\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_rss{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_cache{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_swap{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "node", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, node)", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Kubernetes / Compute Resources / Node (Pods)", + "uid": "200ac8fdbfbb74b39aff88118e4d1c2c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml new file mode 100755 index 000000000..3ec9e37fb --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml @@ -0,0 +1,1772 @@ +{{- /* +Generated from 'k8s-resources-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-pod.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "fill": 0, + "hideTooltip": true, + "legend": true, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", cluster=\"$cluster\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"})\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"})\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 2, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\", cluster=\"$cluster\"}[5m])) by (container) /sum(increase(container_cpu_cfs_periods_total{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\", cluster=\"$cluster\"}[5m])) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0.25, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Throttling", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Throttling", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + }, + { + "alias": "limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\", image!=\"\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"})\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"})\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\", image!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$__interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$__interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$__interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$__interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$__interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$__interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Kubernetes / Compute Resources / Pod", + "uid": "6581e46e4e5c7ba40a07646395ef7b23", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml new file mode 100755 index 000000000..261d3d759 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml @@ -0,0 +1,2034 @@ +{{- /* +Generated from 'k8s-resources-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workload.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Received", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Transmitted", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload)", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\"}, workload_type)", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Kubernetes / Compute Resources / Workload", + "uid": "a164a7f0339f99e89cea5cb47e9be617", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml new file mode 100755 index 000000000..a4b55fc8d --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml @@ -0,0 +1,2195 @@ +{{- /* +Generated from 'k8s-resources-workloads-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workloads-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workloads-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "./d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "quota - requests", + "color": "#F2495C", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + }, + { + "alias": "quota - limits", + "color": "#FF9830", + "dashes": true, + "fill": 0, + "hideTooltip": true, + "legend": false, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - requests", + "legendLink": null, + "step": 10 + }, + { + "expr": "scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "quota - limits", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "./d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "interval": "1m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": false, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$type", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Received", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Transmitted", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Kubernetes / Compute Resources / Namespace (Workloads)", + "uid": "a87fb0d919ec0ea5f6543124e16c42a5", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/kubelet.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/kubelet.yaml new file mode 100755 index 000000000..1775d61ae --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/kubelet.yaml @@ -0,0 +1,2515 @@ +{{- /* +Generated from 'kubelet' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.kubelet.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "kubelet" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + kubelet.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(up{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Running Pods", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Running Container", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\", state=\"actual_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Actual Volume Count", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 6, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",state=\"desired_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Desired Volume Count", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 7, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Config Error Count", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (operation_type, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation Error Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Operation duration 99th quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} pod", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} worker", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Error Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_name{{`}}`}} {{`{{`}}volume_plugin{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Duration 99th quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[5m])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager operation rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}operation_type{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager 99th quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Pod lifecycle event generator", + "fill": 1, + "gridPos": { + + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist interval", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 21, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[5m])) by (instance, verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Request duration 99th quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 23, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 25, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(kubelet_runtime_operations_total{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Kubernetes / Kubelet", + "uid": "3138fa155d5915769fbded898ac09fd9", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/namespace-by-pod.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/namespace-by-pod.yaml new file mode 100755 index 000000000..616e8e01f --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/namespace-by-pod.yaml @@ -0,0 +1,1429 @@ +{{- /* +Generated from 'namespace-by-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-pod.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "pod", + "value": "pod" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/7a18067ce943a40ae25454675c19ff5c/kubernetes-networking-pod?orgId=1&refresh=30s&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total, namespace)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Kubernetes / Networking / Namespace (Pods)", + "uid": "8b7a8b326d7a6f1f04244066368c67af", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/namespace-by-workload.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/namespace-by-workload.yaml new file mode 100755 index 000000000..c7448df22 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/namespace-by-workload.yaml @@ -0,0 +1,1697 @@ +{{- /* +Generated from 'namespace-by-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "namespace-by-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + namespace-by-workload.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "columns": [ + { + "text": "Time", + "value": "Time" + }, + { + "text": "Value #A", + "value": "Value #A" + }, + { + "text": "Value #B", + "value": "Value #B" + }, + { + "text": "Value #C", + "value": "Value #C" + }, + { + "text": "Value #D", + "value": "Value #D" + }, + { + "text": "Value #E", + "value": "Value #E" + }, + { + "text": "Value #F", + "value": "Value #F" + }, + { + "text": "Value #G", + "value": "Value #G" + }, + { + "text": "Value #H", + "value": "Value #H" + }, + { + "text": "workload", + "value": "workload" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/728bf77cc1166d2f3133bf25846876cc/kubernetes-networking-workload?orgId=1&refresh=30s&var-namespace=$namespace&var-type=$type&var-workload=$__cell", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Status", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} workload {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 12, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 15, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 17, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}workload{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total, namespace)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total, namespace)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Kubernetes / Networking / Namespace (Workload)", + "uid": "bbb2a765a623ae38130206c7d94a160f", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml new file mode 100755 index 000000000..4c5c2ac97 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml @@ -0,0 +1,964 @@ +{{- /* +Generated from 'node-cluster-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.nodeExporter.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-cluster-rsrc-use.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_cpu_utilisation:rate1m{job=\"node-exporter\"}\n*\n instance:node_num_cpu:sum{job=\"node-exporter\"}\n)\n/ scalar(sum(instance:node_num_cpu:sum{job=\"node-exporter\"}))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_load1_per_cpu:ratio{job=\"node-exporter\"}\n/ scalar(count(instance:node_load1_per_cpu:ratio{job=\"node-exporter\"}))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (load1 per CPU)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_memory_utilisation:ratio{job=\"node-exporter\"}\n/ scalar(count(instance:node_memory_utilisation:ratio{job=\"node-exporter\"}))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate1m{job=\"node-exporter\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate1m{job=\"node-exporter\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate1m{job=\"node-exporter\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Net Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate1m{job=\"node-exporter\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Receive", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate1m{job=\"node-exporter\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Transmit", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Net Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_seconds:rate1m{job=\"node-exporter\"}\n/ scalar(count(instance_device:node_disk_io_time_seconds:rate1m{job=\"node-exporter\"}))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_weighted_seconds:rate1m{job=\"node-exporter\"}\n/ scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate1m{job=\"node-exporter\"}))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} {{`{{`}}device{{`}}`}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without (device) (\n max without (fstype, mountpoint) (\n node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\"} - node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\"}\n )\n) \n/ scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\"})))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "USE Method / Cluster", + "uid": "3e97d1d02672cdd0861f4c97c64f89b2", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/node-rsrc-use.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/node-rsrc-use.yaml new file mode 100755 index 000000000..6fb2a76ca --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/node-rsrc-use.yaml @@ -0,0 +1,991 @@ +{{- /* +Generated from 'node-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.nodeExporter.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + node-rsrc-use.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_cpu_utilisation:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_load1_per_cpu:ratio{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Saturation", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_memory_utilisation:ratio{job=\"node-exporter\", job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major page faults", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "legendLink": null, + "step": 10 + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Net Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive drops", + "legendLink": null, + "step": 10 + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit drops", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Net Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Net", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_seconds:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_weighted_seconds:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 -\n(\n max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\"})\n/\n max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "instance", + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(up{job=\"node-exporter\"}, instance)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "USE Method / Node", + "uid": "fac67cfbe174d3ef53eb473d73d9212f", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/nodes.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/nodes.yaml new file mode 100755 index 000000000..2357b709b --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/nodes.yaml @@ -0,0 +1,994 @@ +{{- /* +Generated from 'nodes' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"}[$__interval]))\n/ ignoring(cpu) group_left\n count without (cpu)( node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "interval": "1m", + "intervalFactor": 5, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory used", + "refId": "A" + }, + { + "expr": "node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory buffers", + "refId": "B" + }, + { + "expr": "node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory cached", + "refId": "C" + }, + { + "expr": "node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory free", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "100 -\n(\n node_memory_MemAvailable_bytes{job=\"node-exporter\", instance=\"$instance\"}\n/\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n* 100\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Memory Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}} io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "used", + "color": "#E0B400" + }, + { + "alias": "available", + "color": "#73BF69" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n max by (device) (\n node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\"}\n -\n node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\"}\n )\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "used", + "refId": "A" + }, + { + "expr": "sum(\n max by (device) (\n node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\"}\n )\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "available", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_exporter_build_info{job=\"node-exporter\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Nodes", + "uid": "fa49a4706d07a042595b664c87fb33ea", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml new file mode 100755 index 000000000..c73bd7245 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml @@ -0,0 +1,575 @@ +{{- /* +Generated from 'persistentvolumesusage' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "persistentvolumesusage" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + persistentvolumesusage.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n sum without(instance, node) (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used Space", + "refId": "A" + }, + { + "expr": "sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free Space", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume Space Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(\n kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n -\n kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n)\n/\nkubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume Space Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used inodes", + "refId": "A" + }, + { + "expr": "(\n sum without(instance, node) (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": " Free inodes", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume inodes Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n/\nkubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume inodes Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "PersistentVolumeClaim", + "multi": false, + "name": "volume", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\"}, persistentvolumeclaim)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Kubernetes / Persistent Volumes", + "uid": "919b92a8e8041bd567af9edab12c840c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/pod-total.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/pod-total.yaml new file mode 100755 index 000000000..d105b11d6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/pod-total.yaml @@ -0,0 +1,1196 @@ +{{- /* +Generated from 'pod-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "pod-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + pod-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [ + + ], + "override": { + + }, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 8, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total, namespace)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{namespace=~\"$namespace\"}, pod)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total{namespace=~\"$namespace\"}, pod)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Kubernetes / Networking / Pod", + "uid": "7a18067ce943a40ae25454675c19ff5c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml new file mode 100755 index 000000000..d77560793 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/prometheus-remote-write.yaml @@ -0,0 +1,1655 @@ +{{- /* +Generated from 'prometheus-remote-write' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.prometheus.prometheusSpec.remoteWriteDashboards }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus-remote-write" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus-remote-write.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(\n prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"} \n- \n ignoring(remote_name, url) group_right(instance) prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Highest Timestamp In vs. Highest Timestamp Sent", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(\n rate(prometheus_remote_storage_highest_timestamp_in_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m]) \n- \n ignoring (remote_name, url) group_right(instance) rate(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate[5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Timestamps", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(\n prometheus_remote_storage_samples_in_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n ignoring(remote_name, url) group_right(instance) rate(prometheus_remote_storage_succeeded_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n- \n rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate, in vs. succeeded or dropped [5m]", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Samples", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_max{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Max Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_min{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Min Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shards_desired{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Desired Shards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shards", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_shard_capacity{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Shard Capacity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_remote_storage_pending_samples{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Pending Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Shard Details", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_wal_segment_current{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "TSDB Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_wal_watcher_current_segment{cluster=~\"$cluster\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}consumer{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Remote Write Current Segment", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Segments", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_dropped_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Dropped Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_failed_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Failed Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_retried_samples_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Retried Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_remote_storage_enqueue_retries_total{cluster=~\"$cluster\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cluster{{`}}`}}:{{`{{`}}instance{{`}}`}} {{`{{`}}remote_name{{`}}`}}:{{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Enqueue Retries", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Misc. Rates", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "value": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_container_info{image=~\".*prometheus.*\"}, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "url", + "options": [ + + ], + "query": "label_values(prometheus_remote_storage_shards{cluster=~\"$cluster\", instance=~\"$instance\"}, url)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Prometheus Remote Write", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/prometheus.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/prometheus.yaml new file mode 100755 index 000000000..56691a716 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/prometheus.yaml @@ -0,0 +1,1227 @@ +{{- /* +Generated from 'prometheus' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "prometheus" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + prometheus.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Count", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Uptime", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Instance", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "instance", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Job", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "job", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Version", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "version", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count by (job, instance, version) (prometheus_build_info{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "max by (job, instance) (time() - process_start_time_seconds{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Prometheus Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Prometheus Stats", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m])) by (scrape_job) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}scrape_job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Target Sync", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_sd_discovered_targets{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Targets", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Targets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Discovery", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_interval_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}interval{{`}}`}} configured", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Scrape Interval Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded sample limit: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "duplicate timestamp: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of bounds: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of order: {{`{{`}}job{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scrape failures", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Appended Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Retrieval", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head series", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Series", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}} head chunks", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}job{{`}}`}} {{`{{`}}instance{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Query Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",job=~\"$job\",instance=~\"$instance\"}) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}slice{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Stage Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": true, + "name": "job", + "options": [ + + ], + "query": "label_values(prometheus_build_info, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [ + + ], + "query": "label_values(prometheus_build_info, instance)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Prometheus Overview", + "uid": "", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/proxy.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/proxy.yaml new file mode 100755 index 000000000..06d9fdec6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/proxy.yaml @@ -0,0 +1,1222 @@ +{{- /* +Generated from 'proxy' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeProxy.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "proxy" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + proxy.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(up{job=\"{{ include "exporter.kubeProxy.jobName" . }}\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rules Sync Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rule Sync Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\",verb=\"POST\"}[5m])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"{{ include "exporter.kubeProxy.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"{{ include "exporter.kubeProxy.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(kubeproxy_network_programming_duration_seconds_bucket{job=\"{{ include "exporter.kubeProxy.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Kubernetes / Proxy", + "uid": "632e265de029684c40b21cb76bca4f94", + "version": 0 + } +{{- end }}{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/scheduler.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/scheduler.yaml new file mode 100755 index 000000000..e5d7adc49 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/scheduler.yaml @@ -0,0 +1,1068 @@ +{{- /* +Generated from 'scheduler' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeScheduler.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "scheduler" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + scheduler.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(up{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "sum(rate(scheduler_binding_duration_seconds_count{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} e2e", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} binding", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} volume", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}verb{{`}}`}} {{`{{`}}url{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(process_cpu_seconds_total{job=\"{{ include "exporter.kubeScheduler.jobName" . }}\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Kubernetes / Scheduler", + "uid": "2e6b6a3b4bddf1427b3a55aa1311c656", + "version": 0 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/statefulset.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/statefulset.yaml new file mode 100755 index 000000000..abde891ca --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/statefulset.yaml @@ -0,0 +1,927 @@ +{{- /* +Generated from 'statefulset' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "statefulset" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + statefulset.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "cores", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$statefulset.*\"}[3m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "CPU", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "GB", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$statefulset.*\"}) / 1024^3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Memory", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "Bps", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$statefulset.*\"}[3m])) + sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\",pod=~\"$statefulset.*\"}[3m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Network", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "height": "100px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Desired Replicas", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 6, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Replicas of current version", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 7, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(kube_statefulset_status_observed_generation{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Observed Generation", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 8, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Metadata Generation", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "replicas specified", + "refId": "A" + }, + { + "expr": "max(kube_statefulset_status_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "replicas created", + "refId": "B" + }, + { + "expr": "min(kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ready", + "refId": "C" + }, + { + "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "replicas of current version", + "refId": "D" + }, + { + "expr": "min(kube_statefulset_status_replicas_updated{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "updated", + "refId": "E" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Replicas", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_statefulset_metadata_generation, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Name", + "multi": false, + "name": "statefulset", + "options": [ + + ], + "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, statefulset)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Kubernetes / StatefulSets", + "uid": "a31c1f46e6f727cb37c0d731a7245005", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/workload-total.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/workload-total.yaml new file mode 100755 index 000000000..8187a5441 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards-1.14/workload-total.yaml @@ -0,0 +1,1402 @@ +{{- /* +Generated from 'workload-total' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "workload-total" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + workload-total.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Received", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [ + + ], + "minSpan": 24, + "nullPointMode": "null", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 24, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}} pod {{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Rate of Bytes Transmitted", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 8, + "panels": [ + + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 14, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + + ], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "10s", + "rows": [ + + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(container_network_receive_packets_total, namespace)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\"}, workload)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\"}, workload)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(namespace_workload_pod:kube_pod_owner:relabel{namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "UTC", + "title": "Kubernetes / Networking / Workload", + "uid": "728bf77cc1166d2f3133bf25846876cc", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/etcd.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/etcd.yaml new file mode 100755 index 000000000..aa6cc149a --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/etcd.yaml @@ -0,0 +1,1118 @@ +{{- /* +Generated from 'etcd' from https://raw.githubusercontent.com/etcd-io/etcd/master/Documentation/op-guide/grafana.json +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "etcd" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + etcd.json: |- + { + "annotations": { + "list": [] + }, + "description": "etcd sample Grafana dashboard with Prometheus", + "editable": true, + "gnetId": null, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 28, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum(etcd_server_has_leader{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "", + "metric": "etcd_server_has_leader", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "title": "Up", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 23, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{job=\"$cluster\",grpc_type=\"unary\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Rate", + "metric": "grpc_server_started_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(rate(grpc_server_handled_total{job=\"$cluster\",grpc_type=\"unary\",grpc_code!=\"OK\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Failed Rate", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 41, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Watch Streams", + "metric": "grpc_server_handled_total", + "refId": "A", + "step": 4 + }, + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Lease Streams", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "showTitle": false, + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "etcd_mvcc_db_total_size_in_bytes{job=\"$cluster\"}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB Size", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "DB Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$cluster\"}[5m])) by (instance, le))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} WAL fsync", + "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$cluster\"}[5m])) by (instance, le))", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} DB fsync", + "metric": "etcd_disk_backend_commit_duration_seconds_bucket", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 29, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"$cluster\"}", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Resident Memory", + "metric": "process_resident_memory_bytes", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 22, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"$cluster\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic In", + "metric": "etcd_network_client_grpc_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 21, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"$cluster\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Client Traffic Out", + "metric": "etcd_network_client_grpc_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 20, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"$cluster\"}[5m])) by (instance)", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic In", + "metric": "etcd_network_peer_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"$cluster\"}[5m])) by (instance)", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Peer Traffic Out", + "metric": "etcd_network_peer_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 40, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"$cluster\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "Proposal Failure Rate", + "metric": "etcd_server_proposals_failed_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(etcd_server_proposals_pending{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "Proposal Pending Total", + "metric": "etcd_server_proposals_pending", + "refId": "B", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"$cluster\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "Proposal Commit Rate", + "metric": "etcd_server_proposals_committed_total", + "refId": "C", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "Proposal Apply Rate", + "refId": "D", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fill": 0, + "id": 19, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "changes(etcd_server_leader_changes_seen_total{job=\"$cluster\"}[1d])", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}instance{{`}}`}} Total Leader Elections Per Day", + "metric": "etcd_server_leader_changes_seen_total", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Leader Elections Per Day", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + } + ], + "schemaVersion": 13, + "sharedCrosshair": false, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(etcd_server_has_leader, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "etcd", + "uid": "c2f4e12cdf69feb95caa41a5a1b423d9", + "version": 215 + } +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-cluster-rsrc-use.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-cluster-rsrc-use.yaml new file mode 100755 index 000000000..e423526a6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-cluster-rsrc-use.yaml @@ -0,0 +1,959 @@ +{{- /* +Generated from 'k8s-cluster-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-cluster-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-cluster-rsrc-use.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node:cluster_cpu_utilisation:ratio{cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}node{{`}}`}}", + "legendLink": "./d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_cpu_saturation_load1:{cluster=\"$cluster\"} / scalar(sum(min(kube_pod_info{cluster=\"$cluster\"}) by (node)))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}node{{`}}`}}", + "legendLink": "./d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node:cluster_memory_utilisation:ratio{cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}node{{`}}`}}", + "legendLink": "./d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_memory_swap_io_bytes:sum_rate{cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}node{{`}}`}}", + "legendLink": "./d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Swap I/O)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_disk_utilisation:avg_irate{cluster=\"$cluster\"} / scalar(:kube_pod_info_node_count:{cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}node{{`}}`}}", + "legendLink": "./d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_disk_saturation:avg_irate{cluster=\"$cluster\"} / scalar(:kube_pod_info_node_count:{cluster=\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}node{{`}}`}}", + "legendLink": "./d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_net_utilisation:sum_irate{cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}node{{`}}`}}", + "legendLink": "./d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Net Utilisation (Transmitted)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_net_saturation:sum_irate{cluster=\"$cluster\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}node{{`}}`}}", + "legendLink": "./d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Net Saturation (Dropped)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(max(node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\", cluster=\"$cluster\"} - node_filesystem_avail_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\", cluster=\"$cluster\"}) by (device,pod,namespace)) by (pod,namespace)\n/ scalar(sum(max(node_filesystem_size_bytes{fstype=~\"ext[234]|btrfs|xfs|zfs\", cluster=\"$cluster\"}) by (device,pod,namespace)))\n* on (namespace, pod) group_left (node) node_namespace_pod:kube_pod_info:{cluster=\"$cluster\"}\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}node{{`}}`}}", + "legendLink": "./d/4ac4f123aae0ff6dbaf4f4f66120033b/k8s-node-rsrc-use", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Capacity", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(:kube_pod_info_node_count:, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / USE Method / Cluster", + "uid": "a6e7d1362e1ddbb79db21d5bb40d7137", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-node-rsrc-use.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-node-rsrc-use.yaml new file mode 100755 index 000000000..7b1534133 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-node-rsrc-use.yaml @@ -0,0 +1,986 @@ +{{- /* +Generated from 'k8s-node-rsrc-use' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-node-rsrc-use" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-node-rsrc-use.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_cpu_utilisation:avg1m{cluster=\"$cluster\", node=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_cpu_saturation_load1:{cluster=\"$cluster\", node=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Saturation", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_memory_utilisation:{cluster=\"$cluster\", node=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_memory_swap_io_bytes:sum_rate{cluster=\"$cluster\", node=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Swap IO", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Swap I/O)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_disk_utilisation:avg_irate{cluster=\"$cluster\", node=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_disk_saturation:avg_irate{cluster=\"$cluster\", node=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Saturation", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_net_utilisation:sum_irate{cluster=\"$cluster\", node=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Net Utilisation (Transmitted)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_net_saturation:sum_irate{cluster=\"$cluster\", node=\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Saturation", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Net Saturation (Dropped)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Net", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node:node_filesystem_usage:{cluster=\"$cluster\"}\n* on (namespace, pod) group_left (node) node_namespace_pod:kube_pod_info:{cluster=\"$cluster\", node=\"$node\"}\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(:kube_pod_info_node_count:, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "node", + "multi": false, + "name": "node", + "options": [ + + ], + "query": "label_values(kube_node_info{cluster=\"$cluster\"}, node)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / USE Method / Node", + "uid": "4ac4f123aae0ff6dbaf4f4f66120033b", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-cluster.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-cluster.yaml new file mode 100755 index 000000000..1033e8fc9 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-cluster.yaml @@ -0,0 +1,1479 @@ +{{- /* +Generated from 'k8s-resources-cluster' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-cluster" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-cluster.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"$cluster\"}[1m]))", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) / sum(node:node_num_cpu:sum{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Requests Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) / sum(node:node_num_cpu:sum{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Limits Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(:node_memory_MemFreeCachedBuffers_bytes:sum{cluster=\"$cluster\"}) / sum(:node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) / sum(:node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Requests Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) / sum(:node_memory_MemTotal_bytes:sum{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Limits Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to workloads", + "linkUrl": "./d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(mixin_pod_workload{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(mixin_pod_workload{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}namespace{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to workloads", + "linkUrl": "./d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(mixin_pod_workload{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(mixin_pod_workload{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container_name!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests by Namespace", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Requests", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(:kube_pod_info_node_count:, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Cluster", + "uid": "efa86fd1d0c121a26444b636a3f509a8", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-namespace.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-namespace.yaml new file mode 100755 index 000000000..e99dc5bf4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-namespace.yaml @@ -0,0 +1,963 @@ +{{- /* +Generated from 'k8s-resources-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod_name{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"}) by (pod_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod_name{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\",container_name!=\"\"}, \"pod\", \"$1\", \"pod_name\", \"(.*)\")) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(:kube_pod_info_node_count:, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Namespace (Pods)", + "uid": "85a562078cdf77779eaa1add43ccec1e", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-pod.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-pod.yaml new file mode 100755 index 000000000..47291bffd --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-pod.yaml @@ -0,0 +1,1006 @@ +{{- /* +Generated from 'k8s-resources-pod' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-pod" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-pod.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", cluster=\"$cluster\"}) by (container_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container_name{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(label_replace(namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container_name{{`}}`}} (RSS)", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container_name{{`}}`}} (Cache)", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}) by (container_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}container_name{{`}}`}} (Swap)", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"POD\", container_name!=\"\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name!=\"\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(label_replace(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name != \"\", container_name != \"POD\"}, \"container\", \"$1\", \"container_name\", \"(.*)\")) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(:kube_pod_info_node_count:, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "pod", + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Pod", + "uid": "6581e46e4e5c7ba40a07646395ef7b23", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-workload.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-workload.yaml new file mode 100755 index 000000000..3d7cc568c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-workload.yaml @@ -0,0 +1,936 @@ +{{- /* +Generated from 'k8s-resources-workload' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workload" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workload.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n ) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}pod{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n ) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n ) by (pod)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n ) by (pod)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(:kube_pod_info_node_count:, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "workload", + "multi": false, + "name": "workload", + "options": [ + + ], + "query": "label_values(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}, workload)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "type", + "multi": false, + "name": "type", + "options": [ + + ], + "query": "label_values(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\"}, workload_type)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Workload", + "uid": "a164a7f0339f99e89cea5cb47e9be617", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-workloads-namespace.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-workloads-namespace.yaml new file mode 100755 index 000000000..971c84bd8 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/k8s-resources-workloads-namespace.yaml @@ -0,0 +1,972 @@ +{{- /* +Generated from 'k8s-resources-workloads-namespace' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "k8s-resources-workloads-namespace" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + k8s-resources-workloads-namespace.json: |- + { + "annotations": { + "list": [ + + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n ) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}workload{{`}}`}} - {{`{{`}}workload_type{{`}}`}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [ + + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [ + + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n ) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n ) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n label_replace(\n container_memory_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container_name!=\"\"},\n \"pod\", \"$1\", \"pod_name\", \"(.*)\"\n ) * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n ) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod) group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(:kube_pod_info_node_count:, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Namespace (Workloads)", + "uid": "a87fb0d919ec0ea5f6543124e16c42a5", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/nodes.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/nodes.yaml new file mode 100755 index 000000000..9dcd61480 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/nodes.yaml @@ -0,0 +1,1383 @@ +{{- /* +Generated from 'nodes' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "nodes" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + nodes.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(node_load1{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "load 1m", + "refId": "A" + }, + { + "expr": "max(node_load5{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "load 5m", + "refId": "B" + }, + { + "expr": "max(node_load15{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "load 15m", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\", mode=\"user\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "System load", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (cpu) (irate(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}cpu{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Usage Per Core", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": "true", + "avg": "true", + "current": "true", + "max": "false", + "min": "false", + "rightSide": "true", + "show": "true", + "total": "false", + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max (sum by (cpu) (irate(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[2m])) ) * 100\n", + "format": "time_series", + "intervalFactor": 10, + "legendFormat": "{{`{{`}} cpu {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "percent", + "label": null, + "logBase": 1, + "max": 100, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(sum by (cpu) (irate(node_cpu_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", mode!=\"idle\", instance=\"$instance\"}[2m]))) * 100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "CPU Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(\n node_memory_MemTotal_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory used", + "refId": "A" + }, + { + "expr": "max(node_memory_Buffers_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory buffers", + "refId": "B" + }, + { + "expr": "max(node_memory_Cached_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory cached", + "refId": "C" + }, + { + "expr": "max(node_memory_MemFree_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory free", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 7, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(\n (\n (\n node_memory_MemTotal_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_MemFree_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Buffers_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_memory_Cached_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n )\n / node_memory_MemTotal_bytes{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n ) * 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Memory Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "read", + "yaxis": 1 + }, + { + "alias": "io time", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(node_disk_read_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "read", + "refId": "A" + }, + { + "expr": "max(rate(node_disk_written_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "written", + "refId": "B" + }, + { + "expr": "max(rate(node_disk_io_time_seconds_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}[2m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "io time", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max by (namespace, pod, device) ((node_filesystem_size_bytes{cluster=\"$cluster\", fstype=~\"ext[234]|btrfs|xfs|zfs\", instance=\"$instance\", job=\"node-exporter\"} - node_filesystem_avail_bytes{cluster=\"$cluster\", fstype=~\"ext[234]|btrfs|xfs|zfs\", instance=\"$instance\", job=\"node-exporter\"}) / node_filesystem_size_bytes{cluster=\"$cluster\", fstype=~\"ext[234]|btrfs|xfs|zfs\", instance=\"$instance\", job=\"node-exporter\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "disk used", + "refId": "A" + }, + { + "expr": "max by (namespace, pod, device) (node_filesystem_avail_bytes{cluster=\"$cluster\", fstype=~\"ext[234]|btrfs|xfs|zfs\", instance=\"$instance\", job=\"node-exporter\"} / node_filesystem_size_bytes{cluster=\"$cluster\", fstype=~\"ext[234]|btrfs|xfs|zfs\", instance=\"$instance\", job=\"node-exporter\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "disk free", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(node_network_receive_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\", device!~\"lo\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(node_network_transmit_bytes_total{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\", device!~\"lo\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{`{{`}}device{{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(\n node_filesystem_files{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_filesystem_files_free{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "inodes used", + "refId": "A" + }, + { + "expr": "max(node_filesystem_files_free{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "inodes free", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Inodes Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 13, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(\n (\n (\n node_filesystem_files{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n - node_filesystem_files_free{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n )\n / node_filesystem_files{cluster=\"$cluster\", job=\"node-exporter\", instance=\"$instance\"}\n ) * 100)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Inodes Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [ + + ], + "query": "label_values(node_boot_time_seconds{cluster=\"$cluster\", job=\"node-exporter\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Nodes", + "uid": "fa49a4706d07a042595b664c87fb33ea", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/persistentvolumesusage.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/persistentvolumesusage.yaml new file mode 100755 index 000000000..fa6e935ba --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/persistentvolumesusage.yaml @@ -0,0 +1,573 @@ +{{- /* +Generated from 'persistentvolumesusage' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "persistentvolumesusage" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + persistentvolumesusage.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n sum without(instance, node) (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used Space", + "refId": "A" + }, + { + "expr": "sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free Space", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume Space Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(\n kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n -\n kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n)\n/\nkubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume Space Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used inodes", + "refId": "A" + }, + { + "expr": "(\n sum without(instance, node) (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": " Free inodes", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume inodes Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n/\nkubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume inodes Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "PersistentVolumeClaim", + "multi": false, + "name": "volume", + "options": [ + + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\"}, persistentvolumeclaim)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Persistent Volumes", + "uid": "919b92a8e8041bd567af9edab12c840c", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/pods.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/pods.yaml new file mode 100755 index 000000000..e993d5aab --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/pods.yaml @@ -0,0 +1,680 @@ +{{- /* +Generated from 'pods' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "pods" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + pods.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "$datasource", + "enable": true, + "expr": "time() == BOOL timestamp(rate(kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[2m]) > 0)", + "hide": false, + "iconColor": "rgba(215, 44, 44, 1)", + "name": "Restarts", + "showIn": 0, + "tags": [ + "restart" + ], + "type": "rows" + } + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(container_name) (container_memory_usage_bytes{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\", container_name=~\"$container\", container_name!=\"POD\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Current: {{`{{`}} container_name {{`}}`}}", + "refId": "A" + }, + { + "expr": "sum by(container) (kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\", pod=\"$pod\", container=~\"$container\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Requested: {{`{{`}} container {{`}}`}}", + "refId": "B" + }, + { + "expr": "sum by(container) (kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\", pod=\"$pod\", container=~\"$container\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Limit: {{`{{`}} container {{`}}`}}", + "refId": "C" + }, + { + "expr": "sum by(container_name) (container_memory_cache{job=\"kubelet\", namespace=\"$namespace\", pod_name=~\"$pod\", container_name=~\"$container\", container_name!=\"POD\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Cache: {{`{{`}} container_name {{`}}`}}", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (container_name) (rate(container_cpu_usage_seconds_total{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", image!=\"\", pod_name=\"$pod\", container_name=~\"$container\", container_name!=\"POD\"}[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Current: {{`{{`}} container_name {{`}}`}}", + "refId": "A" + }, + { + "expr": "sum by(container) (kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\", pod=\"$pod\", container=~\"$container\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Requested: {{`{{`}} container {{`}}`}}", + "refId": "B" + }, + { + "expr": "sum by(container) (kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\", pod=\"$pod\", container=~\"$container\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Limit: {{`{{`}} container {{`}}`}}", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum by (pod_name) (rate(container_network_receive_bytes_total{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}[1m])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RX: {{`{{`}} pod_name {{`}}`}}", + "refId": "A" + }, + { + "expr": "sort_desc(sum by (pod_name) (rate(container_network_transmit_bytes_total{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=\"$pod\"}[1m])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TX: {{`{{`}} pod_name {{`}}`}}", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max by (container) (kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Restarts: {{`{{`}} container {{`}}`}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Total Restarts Per Container", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Pod", + "multi": false, + "name": "pod", + "options": [ + + ], + "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=~\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Container", + "multi": false, + "name": "container", + "options": [ + + ], + "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}, container)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Pods", + "uid": "ab4f13a9892a76a4d21ce8c2445bf4ea", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/statefulset.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/statefulset.yaml new file mode 100755 index 000000000..643e0d22c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/dashboards/statefulset.yaml @@ -0,0 +1,926 @@ +{{- /* +Generated from 'statefulset' from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/grafana-dashboardDefinitions.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "statefulset" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: + statefulset.json: |- + { + "__inputs": [ + + ], + "__requires": [ + + ], + "annotations": { + "list": [ + + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 2, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "cores", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=~\"$statefulset.*\"}[3m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "CPU", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 3, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "GB", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=~\"$statefulset.*\"}) / 1024^3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Memory", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 4, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "Bps", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(container_network_transmit_bytes_total{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod_name=~\"$statefulset.*\"}[3m])) + sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\",pod_name=~\"$statefulset.*\"}[3m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Network", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "height": "100px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 5, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Desired Replicas", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 6, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Replicas of current version", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 7, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(kube_statefulset_status_observed_generation{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Observed Generation", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + + }, + "id": 8, + "interval": null, + "links": [ + + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Metadata Generation", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "replicas specified", + "refId": "A" + }, + { + "expr": "max(kube_statefulset_status_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "replicas created", + "refId": "B" + }, + { + "expr": "min(kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ready", + "refId": "C" + }, + { + "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "replicas of current version", + "refId": "D" + }, + { + "expr": "min(kube_statefulset_status_replicas_updated{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "updated", + "refId": "E" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeShift": null, + "title": "Replicas", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + + ], + "query": "label_values(kube_statefulset_metadata_generation, cluster)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [ + + ], + "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Name", + "multi": false, + "name": "statefulset", + "options": [ + + ], + "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", namespace=\"$namespace\"}, statefulset)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ + + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / StatefulSets", + "uid": "a31c1f46e6f727cb37c0d731a7245005", + "version": 0 + } +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/namespaces.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/namespaces.yaml new file mode 100755 index 000000000..39ed210ed --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/namespaces.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.grafana.defaultDashboards.namespace }} + labels: + name: {{ .Values.grafana.defaultDashboards.namespace }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + annotations: +{{- if not .Values.grafana.defaultDashboards.cleanupOnUninstall }} + helm.sh/resource-policy: "keep" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/servicemonitor.yaml new file mode 100755 index 000000000..15ee537fb --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/grafana/servicemonitor.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.grafana.enabled .Values.grafana.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-grafana + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-grafana +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.grafana.service.portName }} + {{- if .Values.grafana.serviceMonitor.interval }} + interval: {{ .Values.grafana.serviceMonitor.interval }} + {{- end }} + path: "/metrics" +{{- if .Values.grafana.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.grafana.serviceMonitor.metricRelabelings | indent 6) . }} +{{- end }} +{{- if .Values.grafana.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.grafana.serviceMonitor.relabelings | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml new file mode 100755 index 000000000..7b31ab3f1 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/clusterrole.yaml @@ -0,0 +1,33 @@ +{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission +{{- include "kube-prometheus-stack.labels" $ | indent 4 }} +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - get + - update +{{- if .Values.global.rbac.pspEnabled }} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} + - apiGroups: ['policy'] +{{- else }} + - apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml new file mode 100755 index 000000000..4f1f61665 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission +{{- include "kube-prometheus-stack.labels" $ | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml new file mode 100755 index 000000000..0572e5867 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/job-createSecret.yaml @@ -0,0 +1,65 @@ +{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create +{{- include "kube-prometheus-stack.labels" $ | indent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-create +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-create +{{- include "kube-prometheus-stack.labels" $ | indent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: create + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ template "system_default_registry" . }}{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ template "system_default_registry" . }}{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - create + - --host={{ template "kube-prometheus-stack.operator.fullname" . }},{{ template "kube-prometheus-stack.operator.fullname" . }}.{{ template "kube-prometheus-stack.namespace" . }}.svc + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} +{{- end }} + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml new file mode 100755 index 000000000..991eabf81 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/job-patchWebhook.yaml @@ -0,0 +1,66 @@ +{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch +{{- include "kube-prometheus-stack.labels" $ | indent 4 }} +spec: + {{- if .Capabilities.APIVersions.Has "batch/v1alpha1" }} + # Alpha feature since k8s 1.12 + ttlSecondsAfterFinished: 0 + {{- end }} + template: + metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission-patch +{{- with .Values.prometheusOperator.admissionWebhooks.patch.podAnnotations }} + annotations: +{{ toYaml . | indent 8 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission-patch +{{- include "kube-prometheus-stack.labels" $ | indent 8 }} + spec: + {{- if .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.admissionWebhooks.patch.priorityClassName }} + {{- end }} + containers: + - name: patch + {{- if .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + image: {{ template "system_default_registry" . }}{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }}@sha256:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.sha }} + {{- else }} + image: {{ template "system_default_registry" . }}{{ .Values.prometheusOperator.admissionWebhooks.patch.image.repository }}:{{ .Values.prometheusOperator.admissionWebhooks.patch.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.admissionWebhooks.patch.image.pullPolicy }} + args: + - patch + - --webhook-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --namespace={{ template "kube-prometheus-stack.namespace" . }} + - --secret-name={{ template "kube-prometheus-stack.fullname" . }}-admission + - --patch-failure-policy={{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + resources: +{{ toYaml .Values.prometheusOperator.admissionWebhooks.patch.resources | indent 12 }} + restartPolicy: OnFailure + serviceAccountName: {{ template "kube-prometheus-stack.fullname" . }}-admission + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.admissionWebhooks.patch.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.admissionWebhooks.patch.tolerations }} +{{ toYaml . | indent 8 }} +{{- end }} + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 2000 +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml new file mode 100755 index 000000000..98e002aff --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/psp.yaml @@ -0,0 +1,54 @@ +{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" . }}-admission +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Required to prevent escalations to root. + # allowPrivilegeEscalation: false + # This is redundant with non-root + disallow privilege escalation, + # but we can provide it for defense in depth. + #requiredDropCapabilities: + # - ALL + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml new file mode 100755 index 000000000..3609fe950 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/role.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission +{{- include "kube-prometheus-stack.labels" $ | indent 4 }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml new file mode 100755 index 000000000..dcb0fbcfb --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission +{{- include "kube-prometheus-stack.labels" $ | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kube-prometheus-stack.fullname" . }}-admission +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml new file mode 100755 index 000000000..5296494d3 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/job-patch/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if and .Values.prometheusOperator.admissionWebhooks.enabled .Values.prometheusOperator.admissionWebhooks.patch.enabled .Values.global.rbac.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + namespace: {{ template "kube-prometheus-stack.namespace" . }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission +{{- include "kube-prometheus-stack.labels" $ | indent 4 }} +imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml new file mode 100755 index 000000000..cefee15c2 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/mutatingWebhookConfiguration.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission +{{- include "kube-prometheus-stack.labels" $ | indent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }} + path: /admission-prometheusrules/mutate +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml new file mode 100755 index 000000000..a24270549 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/admission-webhooks/validatingWebhookConfiguration.yaml @@ -0,0 +1,31 @@ +{{- if and .Values.prometheusOperator.admissionWebhooks.enabled }} +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-admission + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-admission +{{- include "kube-prometheus-stack.labels" $ | indent 4 }} +webhooks: + - name: prometheusrulemutate.monitoring.coreos.com + {{- if .Values.prometheusOperator.admissionWebhooks.patch.enabled }} + failurePolicy: Ignore + {{- else }} + failurePolicy: {{ .Values.prometheusOperator.admissionWebhooks.failurePolicy }} + {{- end }} + rules: + - apiGroups: + - monitoring.coreos.com + apiVersions: + - "*" + resources: + - prometheusrules + operations: + - CREATE + - UPDATE + clientConfig: + service: + namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.operator.fullname" $ }} + path: /admission-prometheusrules/validate +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/clusterrole.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/clusterrole.yaml new file mode 100755 index 000000000..1434e076c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/clusterrole.yaml @@ -0,0 +1,79 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-operator + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- if .Values.prometheusOperator.manageCrds }} +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - '*' +{{- end }} +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - thanosrulers + - prometheuses/finalizers + - alertmanagers/finalizers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + verbs: + - '*' +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - '*' +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - delete +- apiGroups: + - "" + resources: + - services + - services/finalizers + - endpoints + verbs: + - get + - create + - update + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/clusterrolebinding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/clusterrolebinding.yaml new file mode 100755 index 000000000..c9ab0ab87 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/clusterrolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-operator + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-operator +subjects: +- kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/deployment.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/deployment.yaml new file mode 100755 index 000000000..de6fb6c0a --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/deployment.yaml @@ -0,0 +1,145 @@ +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +{{- if .Values.prometheusOperator.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-operator + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + template: + metadata: + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{ include "kube-prometheus-stack.labels" . | indent 8 }} +{{- if .Values.prometheusOperator.podLabels }} +{{ toYaml .Values.prometheusOperator.podLabels | indent 8 }} +{{- end }} +{{- if .Values.prometheusOperator.podAnnotations }} + annotations: +{{ toYaml .Values.prometheusOperator.podAnnotations | indent 8 }} +{{- end }} + spec: + {{- if .Values.prometheusOperator.priorityClassName }} + priorityClassName: {{ .Values.prometheusOperator.priorityClassName }} + {{- end }} + containers: + - name: {{ template "kube-prometheus-stack.name" . }} + {{- if .Values.prometheusOperator.image.sha }} + image: "{{ template "system_default_registry" . }}{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag }}@sha256:{{ .Values.prometheusOperator.image.sha }}" + {{- else }} + image: "{{ template "system_default_registry" . }}{{ .Values.prometheusOperator.image.repository }}:{{ .Values.prometheusOperator.image.tag }}" + {{- end }} + imagePullPolicy: "{{ .Values.prometheusOperator.image.pullPolicy }}" + args: + {{- if semverCompare "< v0.39.0" .Values.prometheusOperator.image.tag }} + - --manage-crds={{ .Values.prometheusOperator.manageCrds }} + {{- end }} + {{- if .Values.prometheusOperator.kubeletService.enabled }} + - --kubelet-service={{ .Values.prometheusOperator.kubeletService.namespace }}/{{ template "kube-prometheus-stack.fullname" . }}-kubelet + {{- end }} + {{- if .Values.prometheusOperator.logFormat }} + - --log-format={{ .Values.prometheusOperator.logFormat }} + {{- end }} + {{- if .Values.prometheusOperator.logLevel }} + - --log-level={{ .Values.prometheusOperator.logLevel }} + {{- end }} + {{- if .Values.prometheusOperator.denyNamespaces }} + - --deny-namespaces={{ .Values.prometheusOperator.denyNamespaces | join "," }} + {{- end }} + {{- with $.Values.prometheusOperator.namespaces }} + {{ $ns := .additional }} + {{- if .releaseNamespace }} + {{- $ns = append $ns $namespace }} + {{- end }} + - --namespaces={{ $ns | join "," }} + {{- end }} + - --logtostderr=true + - --localhost=127.0.0.1 + {{- if .Values.prometheusOperator.prometheusConfigReloaderImage.sha }} + - --prometheus-config-reloader={{ template "system_default_registry" . }}{{ .Values.prometheusOperator.prometheusConfigReloaderImage.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloaderImage.tag }}@sha256:{{ .Values.prometheusOperator.prometheusConfigReloaderImage.sha }} + {{- else }} + - --prometheus-config-reloader={{ template "system_default_registry" . }}{{ .Values.prometheusOperator.prometheusConfigReloaderImage.repository }}:{{ .Values.prometheusOperator.prometheusConfigReloaderImage.tag }} + {{- end }} + {{- if .Values.prometheusOperator.configmapReloadImage.sha }} + - --config-reloader-image={{ template "system_default_registry" . }}{{ .Values.prometheusOperator.configmapReloadImage.repository }}:{{ .Values.prometheusOperator.configmapReloadImage.tag }}@sha256:{{ .Values.prometheusOperator.configmapReloadImage.sha }} + {{- else }} + - --config-reloader-image={{ template "system_default_registry" . }}{{ .Values.prometheusOperator.configmapReloadImage.repository }}:{{ .Values.prometheusOperator.configmapReloadImage.tag }} + {{- end }} + - --config-reloader-cpu={{ .Values.prometheusOperator.configReloaderCpu }} + - --config-reloader-memory={{ .Values.prometheusOperator.configReloaderMemory }} + {{- if .Values.prometheusOperator.secretFieldSelector }} + - --secret-field-selector={{ .Values.prometheusOperator.secretFieldSelector }} + {{- end }} + ports: + - containerPort: 8080 + name: http + resources: +{{ toYaml .Values.prometheusOperator.resources | indent 12 }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + {{- if .Values.prometheusOperator.tlsProxy.enabled }} + - name: tls-proxy + {{- if .Values.prometheusOperator.tlsProxy.image.sha }} + image: {{ template "system_default_registry" . }}{{ .Values.prometheusOperator.tlsProxy.image.repository }}:{{ .Values.prometheusOperator.tlsProxy.image.tag }}@sha256:{{ .Values.prometheusOperator.tlsProxy.image.sha }} + {{- else }} + image: {{ template "system_default_registry" . }}{{ .Values.prometheusOperator.tlsProxy.image.repository }}:{{ .Values.prometheusOperator.tlsProxy.image.tag }} + {{- end }} + imagePullPolicy: {{ .Values.prometheusOperator.tlsProxy.image.pullPolicy }} + args: + - server + - --listen=:8443 + - --target=127.0.0.1:8080 + - --key=cert/key + - --cert=cert/cert + - --disable-authentication + resources: +{{ toYaml .Values.prometheusOperator.tlsProxy.resources | indent 12 }} + volumeMounts: + - name: tls-proxy-secret + mountPath: /cert + readOnly: true + ports: + - containerPort: 8443 + name: https + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + {{- end }} +{{- if .Values.prometheusOperator.tlsProxy.enabled }} + volumes: + - name: tls-proxy-secret + secret: + defaultMode: 420 + secretName: {{ template "kube-prometheus-stack.fullname" . }}-admission +{{- end }} +{{- if .Values.prometheusOperator.securityContext }} + securityContext: +{{ toYaml .Values.prometheusOperator.securityContext | indent 8 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} +{{- if .Values.prometheusOperator.hostNetwork }} + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- with .Values.prometheusOperator.nodeSelector }} +{{ toYaml . | indent 8 }} +{{- end }} + {{- with .Values.prometheusOperator.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- with .Values.prometheusOperator.tolerations }} +{{ toYaml . | indent 8 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/psp-clusterrole.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/psp-clusterrole.yaml new file mode 100755 index 000000000..d667d6275 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/psp-clusterrole.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-operator-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-operator +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/psp-clusterrolebinding.yaml new file mode 100755 index 000000000..c538cd173 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/psp-clusterrolebinding.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-operator-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-operator-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/psp.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/psp.yaml new file mode 100755 index 000000000..18d1d37df --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/psp.yaml @@ -0,0 +1,51 @@ +{{- if and .Values.prometheusOperator.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-operator + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Required to prevent escalations to root. + # allowPrivilegeEscalation: false + # This is redundant with non-root + disallow privilege escalation, + # but we can provide it for defense in depth. + #requiredDropCapabilities: + # - ALL + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: {{ .Values.prometheusOperator.hostNetwork }} + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/service.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/service.yaml new file mode 100755 index 000000000..6e67b3d52 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/service.yaml @@ -0,0 +1,53 @@ +{{- if .Values.prometheusOperator.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-operator + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheusOperator.service.labels }} +{{ toYaml .Values.prometheusOperator.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.annotations }} + annotations: +{{ toYaml .Values.prometheusOperator.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheusOperator.service.clusterIP }} + clusterIP: {{ .Values.prometheusOperator.service.clusterIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheusOperator.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheusOperator.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheusOperator.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheusOperator.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} + ports: + - name: http + {{- if eq .Values.prometheusOperator.service.type "NodePort" }} + nodePort: {{ .Values.prometheusOperator.service.nodePort }} + {{- end }} + port: 8080 + targetPort: http + {{- if .Values.prometheusOperator.tlsProxy.enabled }} + - name: https + {{- if eq .Values.prometheusOperator.service.type "NodePort"}} + nodePort: {{ .Values.prometheusOperator.service.nodePortTls }} + {{- end }} + port: 443 + targetPort: https + {{- end }} + selector: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + type: "{{ .Values.prometheusOperator.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/serviceaccount.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/serviceaccount.yaml new file mode 100755 index 000000000..ab41797e3 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.operator.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/servicemonitor.yaml new file mode 100755 index 000000000..52f2d572f --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus-operator/servicemonitor.yaml @@ -0,0 +1,32 @@ +{{- if and .Values.prometheusOperator.enabled .Values.prometheusOperator.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-operator + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-operator +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + endpoints: + - port: http + honorLabels: true + {{- if .Values.prometheusOperator.serviceMonitor.interval }} + interval: {{ .Values.prometheusOperator.serviceMonitor.interval }} + {{- end }} +{{- if .Values.prometheusOperator.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.prometheusOperator.serviceMonitor.metricRelabelings | indent 6) . }} +{{- end }} +{{- if .Values.prometheusOperator.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheusOperator.serviceMonitor.relabelings | indent 6 }} +{{- end }} + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-operator + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/additionalAlertRelabelConfigs.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/additionalAlertRelabelConfigs.yaml new file mode 100755 index 000000000..bff930981 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/additionalAlertRelabelConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-relabel-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alert-relabel-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/additionalAlertmanagerConfigs.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/additionalAlertmanagerConfigs.yaml new file mode 100755 index 000000000..8aebc96c3 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/additionalAlertmanagerConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-am-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-alertmanager-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/additionalPrometheusRules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/additionalPrometheusRules.yaml new file mode 100755 index 000000000..794e9ad27 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/additionalPrometheusRules.yaml @@ -0,0 +1,40 @@ +{{- if or .Values.additionalPrometheusRules .Values.additionalPrometheusRulesMap}} +apiVersion: v1 +kind: List +items: +{{- if .Values.additionalPrometheusRulesMap }} +{{- range $prometheusRuleName, $prometheusRule := .Values.additionalPrometheusRulesMap }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ $prometheusRuleName }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $prometheusRule.additionalLabels }} +{{ toYaml $prometheusRule.additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml $prometheusRule.groups| indent 8 }} +{{- end }} +{{- else }} +{{- range .Values.additionalPrometheusRules }} + - apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: {{ template "kube-prometheus-stack.name" $ }}-{{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }} +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + groups: +{{ toYaml .groups| indent 8 }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/additionalScrapeConfigs.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/additionalScrapeConfigs.yaml new file mode 100755 index 000000000..243124ab9 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/additionalScrapeConfigs.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- if .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations }} + annotations: +{{ toYaml .Values.prometheus.prometheusSpec.additionalPrometheusSecretsAnnotations | indent 4 }} +{{- end }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus-scrape-confg +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +data: + additional-scrape-configs.yaml: {{ toYaml .Values.prometheus.prometheusSpec.additionalScrapeConfigs | b64enc | quote }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/clusterrole.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/clusterrole.yaml new file mode 100755 index 000000000..9772fa5e5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/clusterrole.yaml @@ -0,0 +1,36 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +- apiGroups: + - "" + resources: + - nodes/metrics + verbs: + - get + - list + - watch +# This permission are not in the kube-prometheus repo +# they're grabbed from https://github.com/prometheus/prometheus/blob/master/documentation/examples/rbac-setup.yml +- apiGroups: [""] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: + - extensions + - "networking.k8s.io" + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/clusterrolebinding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/clusterrolebinding.yaml new file mode 100755 index 000000000..9fc4f65da --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} + diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/ingress.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/ingress.yaml new file mode 100755 index 000000000..582d4c585 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/ingress.yaml @@ -0,0 +1,53 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.ingress.enabled }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} +{{- $servicePort := .Values.prometheus.service.port -}} +{{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }} +{{- $paths := .Values.prometheus.ingress.paths | default $routePrefix -}} +{{- if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} +apiVersion: networking.k8s.io/v1beta1 +{{ else }} +apiVersion: extensions/v1beta1 +{{ end -}} +kind: Ingress +metadata: +{{- if .Values.prometheus.ingress.annotations }} + annotations: +{{ toYaml .Values.prometheus.ingress.annotations | indent 4 }} +{{- end }} + name: {{ $serviceName }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.ingress.labels }} +{{ toYaml .Values.prometheus.ingress.labels | indent 4 }} +{{- end }} +spec: + rules: + {{- if .Values.prometheus.ingress.hosts }} + {{- range $host := .Values.prometheus.ingress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + backend: + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + backend: + serviceName: {{ $serviceName }} + servicePort: {{ $servicePort }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.ingress.tls }} + tls: +{{ toYaml .Values.prometheus.ingress.tls | indent 4 }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/ingressThanosSidecar.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/ingressThanosSidecar.yaml new file mode 100755 index 000000000..24d2213dc --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/ingressThanosSidecar.yaml @@ -0,0 +1,48 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.thanosIngress.enabled }} +{{- $serviceName := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} +{{- $thanosPort := .Values.prometheus.thanosIngress.servicePort -}} +{{- $routePrefix := list .Values.prometheus.prometheusSpec.routePrefix }} +{{- $paths := .Values.prometheus.thanosIngress.paths | default $routePrefix -}} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: +{{- if .Values.prometheus.thanosIngress.annotations }} + annotations: +{{ toYaml .Values.prometheus.thanosIngress.annotations | indent 4 }} +{{- end }} + name: {{ template "kube-prometheus-stack.fullname" . }}-thanos-gateway + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.thanosIngress.labels }} +{{ toYaml .Values.prometheus.thanosIngress.labels | indent 4 }} +{{- end }} +spec: + rules: + {{- if .Values.prometheus.thanosIngress.hosts }} + {{- range $host := .Values.prometheus.thanosIngress.hosts }} + - host: {{ tpl $host $ }} + http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + backend: + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end -}} + {{- end -}} + {{- else }} + - http: + paths: + {{- range $p := $paths }} + - path: {{ tpl $p $ }} + backend: + serviceName: {{ $serviceName }} + servicePort: {{ $thanosPort }} + {{- end -}} + {{- end -}} + {{- if .Values.prometheus.thanosIngress.tls }} + tls: +{{ toYaml .Values.prometheus.thanosIngress.tls | indent 4 }} + {{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/ingressperreplica.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/ingressperreplica.yaml new file mode 100755 index 000000000..a7feff016 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/ingressperreplica.yaml @@ -0,0 +1,53 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled .Values.prometheus.ingressPerReplica.enabled }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $servicePort := .Values.prometheus.servicePerReplica.port -}} +{{- $ingressValues := .Values.prometheus.ingressPerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-ingressperreplica + namespace: {{ template "kube-prometheus-stack.namespace" $ }} +items: +{{ range $i, $e := until $count }} + - kind: Ingress + {{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" }} + apiVersion: networking.k8s.io/v1beta1 + {{ else }} + apiVersion: extensions/v1beta1 + {{ end -}} + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $ingressValues.labels }} + {{ toYaml $ingressValues.labels | indent 8 }} + {{- end }} + {{- if $ingressValues.annotations }} + annotations: +{{ toYaml $ingressValues.annotations | indent 8 }} + {{- end }} + spec: + rules: + - host: {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + http: + paths: + {{- range $p := $ingressValues.paths }} + - path: {{ tpl $p $ }} + backend: + serviceName: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + servicePort: {{ $servicePort }} + {{- end -}} + {{- if or $ingressValues.tlsSecretName $ingressValues.tlsSecretPerReplica.enabled }} + tls: + - hosts: + - {{ $ingressValues.hostPrefix }}-{{ $i }}.{{ $ingressValues.hostDomain }} + {{- if $ingressValues.tlsSecretPerReplica.enabled }} + secretName: {{ $ingressValues.tlsSecretPerReplica.prefix }}-{{ $i }} + {{- else }} + secretName: {{ $ingressValues.tlsSecretName }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/nginx-config.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/nginx-config.yaml new file mode 100755 index 000000000..bc169a72d --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/nginx-config.yaml @@ -0,0 +1,66 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-nginx-proxy-config + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +data: + nginx.conf: |- + worker_processes auto; + error_log /dev/stdout warn; + pid /var/cache/nginx/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + log_format main '[$time_local - $status] $remote_addr - $remote_user $request ($http_referer)'; + + proxy_connect_timeout 10; + proxy_read_timeout 180; + proxy_send_timeout 5; + proxy_buffering off; + proxy_cache_path /var/cache/nginx/cache levels=1:2 keys_zone=my_zone:100m inactive=1d max_size=10g; + + server { + listen 8080; + access_log off; + + gzip on; + gzip_min_length 1k; + gzip_comp_level 2; + gzip_types text/plain application/javascript application/x-javascript text/css application/xml text/javascript image/jpeg image/gif image/png; + gzip_vary on; + gzip_disable "MSIE [1-6]\."; + + proxy_set_header Host $host; + + location / { + proxy_cache my_zone; + proxy_cache_valid 200 302 1d; + proxy_cache_valid 301 30d; + proxy_cache_valid any 5m; + proxy_cache_bypass $http_cache_control; + add_header X-Proxy-Cache $upstream_cache_status; + add_header Cache-Control "public"; + + proxy_pass http://localhost:9090/; + + sub_filter_types text/html; + sub_filter_once off; + sub_filter 'var PATH_PREFIX = "";' 'var PATH_PREFIX = ".";'; + + if ($request_filename ~ .*\.(?:js|css|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm)$) { + expires 90d; + } + } + } + } diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/podDisruptionBudget.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/podDisruptionBudget.yaml new file mode 100755 index 000000000..573317a32 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/podDisruptionBudget.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.podDisruptionBudget.enabled }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + {{- if .Values.prometheus.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.prometheus.podDisruptionBudget.minAvailable }} + {{- end }} + {{- if .Values.prometheus.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.prometheus.podDisruptionBudget.maxUnavailable }} + {{- end }} + selector: + matchLabels: + app: prometheus + prometheus: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/podmonitors.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/podmonitors.yaml new file mode 100755 index 000000000..95d568e13 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/podmonitors.yaml @@ -0,0 +1,37 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalPodMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalPodMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: PodMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + podMetricsEndpoints: +{{ toYaml .podMetricsEndpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .podTargetLabels }} + podTargetLabels: +{{ toYaml .podTargetLabels | indent 8 }} + {{- end }} + {{- if .sampleLimit }} + sampleLimit: {{ .sampleLimit }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/prometheus.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/prometheus.yaml new file mode 100755 index 000000000..960d16d43 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/prometheus.yaml @@ -0,0 +1,262 @@ +{{- if .Values.prometheus.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.annotations }} + annotations: +{{ toYaml .Values.prometheus.annotations | indent 4 }} +{{- end }} +spec: + alerting: + alertmanagers: +{{- if .Values.prometheus.prometheusSpec.alertingEndpoints }} +{{ toYaml .Values.prometheus.prometheusSpec.alertingEndpoints | indent 6 }} +{{- else if .Values.alertmanager.enabled }} + - namespace: {{ template "kube-prometheus-stack.namespace" . }} + name: {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + port: {{ .Values.alertmanager.alertmanagerSpec.portName }} + {{- if .Values.alertmanager.alertmanagerSpec.routePrefix }} + pathPrefix: "{{ .Values.alertmanager.alertmanagerSpec.routePrefix }}" + {{- end }} + apiVersion: {{ .Values.alertmanager.apiVersion }} +{{- else }} + [] +{{- end }} +{{- if .Values.prometheus.prometheusSpec.apiserverConfig }} + apiserverConfig: +{{ toYaml .Values.prometheus.prometheusSpec.apiserverConfig | indent 4}} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.image }} + baseImage: {{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.image.repository }} + version: {{ .Values.prometheus.prometheusSpec.image.tag }} + {{- if .Values.prometheus.prometheusSpec.image.sha }} + sha: {{ .Values.prometheus.prometheusSpec.image.sha }} + {{- end }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.externalLabels }} + externalLabels: +{{ toYaml .Values.prometheus.prometheusSpec.externalLabels | indent 4}} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.prometheusExternalLabelNameClear }} + prometheusExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.prometheusExternalLabelName }} + prometheusExternalLabelName: "{{ .Values.prometheus.prometheusSpec.prometheusExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.replicaExternalLabelNameClear }} + replicaExternalLabelName: "" +{{- else if .Values.prometheus.prometheusSpec.replicaExternalLabelName }} + replicaExternalLabelName: "{{ .Values.prometheus.prometheusSpec.replicaExternalLabelName }}" +{{- end }} +{{- if .Values.prometheus.prometheusSpec.externalUrl }} + externalUrl: "{{ tpl .Values.prometheus.prometheusSpec.externalUrl . }}" +{{- else if and .Values.prometheus.ingress.enabled .Values.prometheus.ingress.hosts }} + externalUrl: "http://{{ tpl (index .Values.prometheus.ingress.hosts 0) . }}{{ .Values.prometheus.prometheusSpec.routePrefix }}" +{{- else }} + externalUrl: http://{{ template "kube-prometheus-stack.fullname" . }}-prometheus.{{ template "kube-prometheus-stack.namespace" . }}:{{ .Values.prometheus.service.port }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} + ignoreNamespaceSelectors: {{ .Values.prometheus.prometheusSpec.ignoreNamespaceSelectors }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.nodeSelector }} +{{ toYaml .Values.prometheus.prometheusSpec.nodeSelector | indent 4 }} +{{- end }} + paused: {{ .Values.prometheus.prometheusSpec.paused }} + replicas: {{ .Values.prometheus.prometheusSpec.replicas }} + logLevel: {{ .Values.prometheus.prometheusSpec.logLevel }} + logFormat: {{ .Values.prometheus.prometheusSpec.logFormat }} + listenLocal: {{ .Values.prometheus.prometheusSpec.listenLocal }} + enableAdminAPI: {{ .Values.prometheus.prometheusSpec.enableAdminAPI }} +{{- if .Values.prometheus.prometheusSpec.scrapeInterval }} + scrapeInterval: {{ .Values.prometheus.prometheusSpec.scrapeInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.evaluationInterval }} + evaluationInterval: {{ .Values.prometheus.prometheusSpec.evaluationInterval }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.resources }} + resources: +{{ toYaml .Values.prometheus.prometheusSpec.resources | indent 4 }} +{{- end }} + retention: {{ .Values.prometheus.prometheusSpec.retention | quote }} +{{- if .Values.prometheus.prometheusSpec.retentionSize }} + retentionSize: {{ .Values.prometheus.prometheusSpec.retentionSize | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.walCompression }} + walCompression: {{ .Values.prometheus.prometheusSpec.walCompression }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.routePrefix }} + routePrefix: {{ .Values.prometheus.prometheusSpec.routePrefix | quote }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.secrets }} + secrets: +{{ toYaml .Values.prometheus.prometheusSpec.secrets | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.configMaps }} + configMaps: +{{ toYaml .Values.prometheus.prometheusSpec.configMaps | indent 4 }} +{{- end }} + serviceAccountName: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorSelector }} + serviceMonitorSelector: +{{ toYaml .Values.prometheus.prometheusSpec.serviceMonitorSelector | indent 4 }} +{{ else if .Values.prometheus.prometheusSpec.serviceMonitorSelectorNilUsesHelmValues }} + serviceMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + serviceMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector }} + serviceMonitorNamespaceSelector: +{{ toYaml .Values.prometheus.prometheusSpec.serviceMonitorNamespaceSelector | indent 4 }} +{{ else }} + serviceMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorSelector }} + podMonitorSelector: +{{ toYaml .Values.prometheus.prometheusSpec.podMonitorSelector | indent 4 }} +{{ else if .Values.prometheus.prometheusSpec.podMonitorSelectorNilUsesHelmValues }} + podMonitorSelector: + matchLabels: + release: {{ $.Release.Name | quote }} +{{ else }} + podMonitorSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector }} + podMonitorNamespaceSelector: +{{ toYaml .Values.prometheus.prometheusSpec.podMonitorNamespaceSelector | indent 4 }} +{{ else }} + podMonitorNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.remoteRead }} + remoteRead: +{{ toYaml .Values.prometheus.prometheusSpec.remoteRead | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.remoteWrite }} + remoteWrite: +{{ toYaml .Values.prometheus.prometheusSpec.remoteWrite | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.securityContext }} + securityContext: +{{ toYaml .Values.prometheus.prometheusSpec.securityContext | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ruleNamespaceSelector }} + ruleNamespaceSelector: +{{ toYaml .Values.prometheus.prometheusSpec.ruleNamespaceSelector | indent 4 }} +{{ else }} + ruleNamespaceSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.ruleSelector }} + ruleSelector: +{{ toYaml .Values.prometheus.prometheusSpec.ruleSelector | indent 4}} +{{- else if .Values.prometheus.prometheusSpec.ruleSelectorNilUsesHelmValues }} + ruleSelector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }} + release: {{ $.Release.Name | quote }} +{{ else }} + ruleSelector: {} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.storageSpec }} + storage: +{{ toYaml .Values.prometheus.prometheusSpec.storageSpec | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.podMetadata }} + podMetadata: +{{ toYaml .Values.prometheus.prometheusSpec.podMetadata | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.query }} + query: +{{ toYaml .Values.prometheus.prometheusSpec.query | indent 4}} +{{- end }} +{{- if or .Values.prometheus.prometheusSpec.podAntiAffinity .Values.prometheus.prometheusSpec.affinity }} + affinity: +{{- if .Values.prometheus.prometheusSpec.affinity }} +{{ toYaml .Values.prometheus.prometheusSpec.affinity | indent 4 }} +{{- end }} +{{- if eq .Values.prometheus.prometheusSpec.podAntiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchLabels: + app: prometheus + prometheus: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- else if eq .Values.prometheus.prometheusSpec.podAntiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: {{ .Values.prometheus.prometheusSpec.podAntiAffinityTopologyKey }} + labelSelector: + matchLabels: + app: prometheus + prometheus: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 4 }} +{{- if .Values.prometheus.prometheusSpec.tolerations }} +{{ toYaml .Values.prometheus.prometheusSpec.tolerations | indent 4 }} +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigs }} + additionalScrapeConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-scrape-confg + key: additional-scrape-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.enabled }} + additionalScrapeConfigs: + name: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.name }} + key: {{ .Values.prometheus.prometheusSpec.additionalScrapeConfigsSecret.key }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertManagerConfigs }} + additionalAlertManagerConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-confg + key: additional-alertmanager-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.additionalAlertRelabelConfigs }} + additionalAlertRelabelConfigs: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-am-relabel-confg + key: additional-alert-relabel-configs.yaml +{{- end }} +{{- if .Values.prometheus.prometheusSpec.containers }} + containers: +{{ tpl .Values.prometheus.prometheusSpec.containers $ | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.initContainers }} + initContainers: +{{ toYaml .Values.prometheus.prometheusSpec.initContainers | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.priorityClassName }} + priorityClassName: {{ .Values.prometheus.prometheusSpec.priorityClassName }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.thanos }} + thanos: +{{ toYaml .Values.prometheus.prometheusSpec.thanos | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.disableCompaction }} + disableCompaction: {{ .Values.prometheus.prometheusSpec.disableCompaction }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.portName }} + portName: {{ .Values.prometheus.prometheusSpec.portName }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} + enforcedNamespaceLabel: {{ .Values.prometheus.prometheusSpec.enforcedNamespaceLabel }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.volumes }} + volumes: +{{ toYaml .Values.prometheus.prometheusSpec.volumes | indent 4 }} +{{- end }} +{{- if .Values.prometheus.prometheusSpec.volumeMounts }} + volumeMounts: +{{ toYaml .Values.prometheus.prometheusSpec.volumeMounts | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/psp-clusterrole.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/psp-clusterrole.yaml new file mode 100755 index 000000000..a279fb241 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/psp-clusterrole.yaml @@ -0,0 +1,20 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +rules: +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/psp-clusterrolebinding.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/psp-clusterrolebinding.yaml new file mode 100755 index 000000000..27b73b74b --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/psp-clusterrolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus-psp +subjects: + - kind: ServiceAccount + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} +{{- end }} + diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/psp.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/psp.yaml new file mode 100755 index 000000000..bd2b270d5 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/psp.yaml @@ -0,0 +1,55 @@ +{{- if and .Values.prometheus.enabled .Values.global.rbac.create .Values.global.rbac.pspEnabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{- if .Values.global.rbac.pspAnnotations }} + annotations: +{{ toYaml .Values.global.rbac.pspAnnotations | indent 4 }} +{{- end }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + privileged: false + # Required to prevent escalations to root. + # allowPrivilegeEscalation: false + # This is redundant with non-root + disallow privilege escalation, + # but we can provide it for defense in depth. + #requiredDropCapabilities: + # - ALL + # Allow core volume types. + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +{{- if .Values.prometheus.podSecurityPolicy.allowedCapabilities }} + allowedCapabilities: +{{ toYaml .Values.prometheus.podSecurityPolicy.allowedCapabilities | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/alertmanager.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/alertmanager.rules.yaml new file mode 100755 index 000000000..37c22e16a --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/alertmanager.rules.yaml @@ -0,0 +1,61 @@ +{{- /* +Generated from 'alertmanager.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.alertmanager }} +{{- $alertmanagerJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: alertmanager.rules + rules: + - alert: AlertmanagerConfigInconsistent + annotations: + message: 'The configuration of the instances of the Alertmanager cluster `{{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.service {{`}}`}}` are out of sync. + + {{`{{`}} range printf "alertmanager_config_hash{namespace=\"%s\",service=\"%s\"}" $labels.namespace $labels.service | query {{`}}`}} + + Configuration hash for pod {{`{{`}} .Labels.pod {{`}}`}} is "{{`{{`}} printf "%.f" .Value {{`}}`}}" + + {{`{{`}} end {{`}}`}} + + ' + expr: count by(namespace,service) (count_values by(namespace,service) ("config_hash", alertmanager_config_hash{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"})) != 1 + for: 5m + labels: + severity: critical + - alert: AlertmanagerFailedReload + annotations: + message: Reloading Alertmanager's configuration has failed for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}}. + expr: alertmanager_config_last_reload_successful{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} == 0 + for: 10m + labels: + severity: warning + - alert: AlertmanagerMembersInconsistent + annotations: + message: Alertmanager has not found all other members of the cluster. + expr: |- + alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + != on (service) GROUP_LEFT() + count by (service) (alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}) + for: 5m + labels: + severity: critical +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/etcd.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/etcd.yaml new file mode 100755 index 000000000..fbb1720f8 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/etcd.yaml @@ -0,0 +1,157 @@ +{{- /* +Generated from 'etcd' group from https://raw.githubusercontent.com/etcd-io/etcd/master/Documentation/op-guide/etcd3_alert.rules.yml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.etcd }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "etcd" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: etcd + rules: + - alert: etcdMembersDown + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": members are down ({{`{{`}} $value {{`}}`}}).' + expr: |- + max by (job) ( + sum by (job) (up{job=~".*etcd.*"} == bool 0) + or + count by (job,endpoint) ( + sum by (job,endpoint,To) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[3m])) > 0.01 + ) + ) + > 0 + for: 3m + labels: + severity: critical + - alert: etcdInsufficientMembers + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": insufficient members ({{`{{`}} $value {{`}}`}}).' + expr: sum(up{job=~".*etcd.*"} == bool 1) by (job) < ((count(up{job=~".*etcd.*"}) by (job) + 1) / 2) + for: 3m + labels: + severity: critical + - alert: etcdNoLeader + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member {{`{{`}} $labels.instance {{`}}`}} has no leader.' + expr: etcd_server_has_leader{job=~".*etcd.*"} == 0 + for: 1m + labels: + severity: critical + - alert: etcdHighNumberOfLeaderChanges + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.' + expr: increase((max by (job) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 3 + for: 5m + labels: + severity: warning + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method) + > 1 + for: 10m + labels: + severity: warning + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method) + > 5 + for: 5m + labels: + severity: critical + - alert: etcdGRPCRequestsSlow + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": gRPC requests to {{`{{`}} $labels.grpc_method {{`}}`}} are taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + expr: |- + histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le)) + > 0.15 + for: 10m + labels: + severity: critical + - alert: etcdMemberCommunicationSlow + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member communication with {{`{{`}} $labels.To {{`}}`}} is taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + expr: |- + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.15 + for: 10m + labels: + severity: warning + - alert: etcdHighNumberOfFailedProposals + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last 30 minutes on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 + for: 15m + labels: + severity: warning + - alert: etcdHighFsyncDurations + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.5 + for: 10m + labels: + severity: warning + - alert: etcdHighCommitDurations + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile commit durations {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + expr: |- + histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.25 + for: 10m + labels: + severity: warning + - alert: etcdHighNumberOfFailedHTTPRequests + annotations: + message: '{{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}' + expr: |- + sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) + BY (method) > 0.01 + for: 10m + labels: + severity: warning + - alert: etcdHighNumberOfFailedHTTPRequests + annotations: + message: '{{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + expr: |- + sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) + BY (method) > 0.05 + for: 10m + labels: + severity: critical + - alert: etcdHTTPRequestsSlow + annotations: + message: etcd instance {{`{{`}} $labels.instance {{`}}`}} HTTP requests to {{`{{`}} $labels.method {{`}}`}} are slow. + expr: |- + histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m])) + > 0.15 + for: 10m + labels: + severity: warning +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/general.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/general.rules.yaml new file mode 100755 index 000000000..e8aadc86d --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/general.rules.yaml @@ -0,0 +1,50 @@ +{{- /* +Generated from 'general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.general }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: general.rules + rules: + - alert: TargetDown + annotations: + message: '{{`{{`}} printf "%.4g" $value {{`}}`}}% of the {{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.service {{`}}`}} targets in {{`{{`}} $labels.namespace {{`}}`}} namespace are down.' + expr: 100 * (count(up == 0) BY (job, namespace, service) / count(up) BY (job, namespace, service)) > 10 + for: 10m + labels: + severity: warning + - alert: Watchdog + annotations: + message: 'This is an alert meant to ensure that the entire alerting pipeline is functional. + + This alert is always firing, therefore it should always be firing in Alertmanager + + and always fire against a receiver. There are integrations with various notification + + mechanisms that send a notification when this alert is not firing. For example the + + "DeadMansSnitch" integration in PagerDuty. + + ' + expr: vector(1) + labels: + severity: none +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/k8s.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/k8s.rules.yaml new file mode 100755 index 000000000..35e4edebf --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/k8s.rules.yaml @@ -0,0 +1,121 @@ +{{- /* +Generated from 'k8s.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8s }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules + rules: + - expr: sum(rate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}[5m])) by (namespace) + record: namespace:container_cpu_usage_seconds_total:sum_rate + - expr: |- + sum by (cluster, namespace, pod, container) ( + rate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}[5m]) + ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) ( + 1, max by(cluster, namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate + - expr: |- + container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, + max by(namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_working_set_bytes + - expr: |- + container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, + max by(namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_rss + - expr: |- + container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, + max by(namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_cache + - expr: |- + container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""} + * on (namespace, pod) group_left(node) topk by(namespace, pod) (1, + max by(namespace, pod, node) (kube_pod_info{node!=""}) + ) + record: node_namespace_pod_container:container_memory_swap + - expr: sum(container_memory_usage_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!="", container!="POD"}) by (namespace) + record: namespace:container_memory_usage_bytes:sum + - expr: |- + sum by (namespace) ( + sum by (namespace, pod) ( + max by (namespace, pod, container) ( + kube_pod_container_resource_requests_memory_bytes{job="kube-state-metrics"} + ) * on(namespace, pod) group_left() max by (namespace, pod) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace:kube_pod_container_resource_requests_memory_bytes:sum + - expr: |- + sum by (namespace) ( + sum by (namespace, pod) ( + max by (namespace, pod, container) ( + kube_pod_container_resource_requests_cpu_cores{job="kube-state-metrics"} + ) * on(namespace, pod) group_left() max by (namespace, pod) ( + kube_pod_status_phase{phase=~"Pending|Running"} == 1 + ) + ) + ) + record: namespace:kube_pod_container_resource_requests_cpu_cores:sum + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on(replicaset, namespace) group_left(owner_name) topk by(replicaset, namespace) ( + 1, max by (replicaset, namespace, owner_name) ( + kube_replicaset_owner{job="kube-state-metrics"} + ) + ), + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: deployment + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: daemonset + record: namespace_workload_pod:kube_pod_owner:relabel + - expr: |- + max by (cluster, namespace, workload, pod) ( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) + labels: + workload_type: statefulset + record: namespace_workload_pod:kube_pod_owner:relabel +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml new file mode 100755 index 000000000..78a09e323 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml @@ -0,0 +1,158 @@ +{{- /* +Generated from 'kube-apiserver-availability.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverAvailability }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-availability.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - interval: 3m + name: kube-apiserver-availability.rules + rules: + - expr: |- + 1 - ( + ( + # write too slow + sum(increase(apiserver_request_duration_seconds_count{verb=~"POST|PUT|PATCH|DELETE"}[30d])) + - + sum(increase(apiserver_request_duration_seconds_bucket{verb=~"POST|PUT|PATCH|DELETE",le="1"}[30d])) + ) + + ( + # read too slow + sum(increase(apiserver_request_duration_seconds_count{verb=~"LIST|GET"}[30d])) + - + ( + ( + sum(increase(apiserver_request_duration_seconds_bucket{verb=~"LIST|GET",scope=~"resource|",le="0.1"}[30d])) + or + vector(0) + ) + + + sum(increase(apiserver_request_duration_seconds_bucket{verb=~"LIST|GET",scope="namespace",le="0.5"}[30d])) + + + sum(increase(apiserver_request_duration_seconds_bucket{verb=~"LIST|GET",scope="cluster",le="5"}[30d])) + ) + ) + + # errors + sum(code:apiserver_request_total:increase30d{code=~"5.."} or vector(0)) + ) + / + sum(code:apiserver_request_total:increase30d) + labels: + verb: all + record: apiserver_request:availability30d + - expr: |- + 1 - ( + sum(increase(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[30d])) + - + ( + # too slow + ( + sum(increase(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[30d])) + or + vector(0) + ) + + + sum(increase(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[30d])) + + + sum(increase(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[30d])) + ) + + + # errors + sum(code:apiserver_request_total:increase30d{verb="read",code=~"5.."} or vector(0)) + ) + / + sum(code:apiserver_request_total:increase30d{verb="read"}) + labels: + verb: read + record: apiserver_request:availability30d + - expr: |- + 1 - ( + ( + # too slow + sum(increase(apiserver_request_duration_seconds_count{verb=~"POST|PUT|PATCH|DELETE"}[30d])) + - + sum(increase(apiserver_request_duration_seconds_bucket{verb=~"POST|PUT|PATCH|DELETE",le="1"}[30d])) + ) + + + # errors + sum(code:apiserver_request_total:increase30d{verb="write",code=~"5.."} or vector(0)) + ) + / + sum(code:apiserver_request_total:increase30d{verb="write"}) + labels: + verb: write + record: apiserver_request:availability30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"2.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"2.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"2.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"2.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"2.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"2.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"3.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"3.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"3.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"3.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"3.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"3.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"4.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"4.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"4.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"4.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"4.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"4.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="LIST",code=~"5.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="GET",code=~"5.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="POST",code=~"5.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PUT",code=~"5.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="PATCH",code=~"5.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code, verb) (increase(apiserver_request_total{job="apiserver",verb="DELETE",code=~"5.."}[30d])) + record: code_verb:apiserver_request_total:increase30d + - expr: sum by (code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"}) + labels: + verb: read + record: code:apiserver_request_total:increase30d + - expr: sum by (code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"}) + labels: + verb: write + record: code:apiserver_request_total:increase30d +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml new file mode 100755 index 000000000..2ac67913f --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml @@ -0,0 +1,79 @@ +{{- /* +Generated from 'kube-apiserver-slos' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverSlos }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver-slos" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver-slos + rules: + - alert: KubeAPIErrorBudgetBurn + annotations: + message: The API server is burning too much error budget + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorbudgetburn + expr: |- + sum(apiserver_request:burnrate1h) > (14.40 * 0.01000) + and + sum(apiserver_request:burnrate5m) > (14.40 * 0.01000) + for: 2m + labels: + long: 1h + severity: critical + short: 5m + - alert: KubeAPIErrorBudgetBurn + annotations: + message: The API server is burning too much error budget + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorbudgetburn + expr: |- + sum(apiserver_request:burnrate6h) > (6.00 * 0.01000) + and + sum(apiserver_request:burnrate30m) > (6.00 * 0.01000) + for: 15m + labels: + long: 6h + severity: critical + short: 30m + - alert: KubeAPIErrorBudgetBurn + annotations: + message: The API server is burning too much error budget + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorbudgetburn + expr: |- + sum(apiserver_request:burnrate1d) > (3.00 * 0.01000) + and + sum(apiserver_request:burnrate2h) > (3.00 * 0.01000) + for: 1h + labels: + long: 1d + severity: warning + short: 2h + - alert: KubeAPIErrorBudgetBurn + annotations: + message: The API server is burning too much error budget + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorbudgetburn + expr: |- + sum(apiserver_request:burnrate3d) > (1.00 * 0.01000) + and + sum(apiserver_request:burnrate6h) > (1.00 * 0.01000) + for: 3h + labels: + long: 3d + severity: warning + short: 6h +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml new file mode 100755 index 000000000..b4d1a0f39 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-apiserver.rules.yaml @@ -0,0 +1,363 @@ +{{- /* +Generated from 'kube-apiserver.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserver }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver.rules + rules: + - expr: |- + ( + ( + # too slow + sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[1d])) + - + ( + ( + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[1d])) + or + vector(0) + ) + + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[1d])) + + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[1d])) + ) + ) + + + # errors + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1d])) + ) + / + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d])) + labels: + verb: read + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[1h])) + - + ( + ( + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[1h])) + or + vector(0) + ) + + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[1h])) + + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[1h])) + ) + ) + + + # errors + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1h])) + ) + / + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h])) + labels: + verb: read + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[2h])) + - + ( + ( + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[2h])) + or + vector(0) + ) + + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[2h])) + + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[2h])) + ) + ) + + + # errors + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[2h])) + ) + / + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h])) + labels: + verb: read + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[30m])) + - + ( + ( + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[30m])) + or + vector(0) + ) + + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[30m])) + + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[30m])) + ) + ) + + + # errors + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[30m])) + ) + / + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m])) + labels: + verb: read + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[3d])) + - + ( + ( + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[3d])) + or + vector(0) + ) + + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[3d])) + + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[3d])) + ) + ) + + + # errors + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[3d])) + ) + / + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d])) + labels: + verb: read + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[5m])) + - + ( + ( + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[5m])) + or + vector(0) + ) + + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[5m])) + + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[5m])) + ) + ) + + + # errors + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[5m])) + ) + / + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"LIST|GET"}[6h])) + - + ( + ( + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope=~"resource|",le="0.1"}[6h])) + or + vector(0) + ) + + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="namespace",le="0.5"}[6h])) + + + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",scope="cluster",le="5"}[6h])) + ) + ) + + + # errors + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[6h])) + ) + / + sum(rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h])) + labels: + verb: read + record: apiserver_request:burnrate6h + - expr: |- + ( + ( + # too slow + sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) + - + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[1d])) + ) + + + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d])) + ) + / + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d])) + labels: + verb: write + record: apiserver_request:burnrate1d + - expr: |- + ( + ( + # too slow + sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) + - + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[1h])) + ) + + + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h])) + ) + / + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h])) + labels: + verb: write + record: apiserver_request:burnrate1h + - expr: |- + ( + ( + # too slow + sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) + - + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[2h])) + ) + + + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h])) + ) + / + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h])) + labels: + verb: write + record: apiserver_request:burnrate2h + - expr: |- + ( + ( + # too slow + sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) + - + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[30m])) + ) + + + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m])) + ) + / + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m])) + labels: + verb: write + record: apiserver_request:burnrate30m + - expr: |- + ( + ( + # too slow + sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) + - + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[3d])) + ) + + + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d])) + ) + / + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d])) + labels: + verb: write + record: apiserver_request:burnrate3d + - expr: |- + ( + ( + # too slow + sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + - + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[5m])) + ) + + + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m])) + ) + / + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + record: apiserver_request:burnrate5m + - expr: |- + ( + ( + # too slow + sum(rate(apiserver_request_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) + - + sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",le="1"}[6h])) + ) + + + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h])) + ) + / + sum(rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h])) + labels: + verb: write + record: apiserver_request:burnrate6h + - expr: sum by (code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m])) + labels: + verb: read + record: code_resource:apiserver_request_total:rate5m + - expr: sum by (code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m])) + labels: + verb: write + record: code_resource:apiserver_request_total:rate5m + - expr: histogram_quantile(0.99, sum by (le, resource) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: read + record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum by (le, resource) (rate(apiserver_request_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))) > 0 + labels: + quantile: '0.99' + verb: write + record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile + - expr: |- + sum(rate(apiserver_request_duration_seconds_sum{subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod) + / + sum(rate(apiserver_request_duration_seconds_count{subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod) + record: cluster:apiserver_request_duration_seconds:mean5m + - expr: histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(apiserver_request_duration_seconds_bucket{job="apiserver",subresource!="log",verb!~"LIST|WATCH|WATCHLIST|DELETECOLLECTION|PROXY|CONNECT"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + record: cluster_quantile:apiserver_request_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml new file mode 100755 index 000000000..e54bee587 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml @@ -0,0 +1,31 @@ +{{- /* +Generated from 'kube-prometheus-general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusGeneral }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-general.rules + rules: + - expr: count without(instance, pod, node) (up == 1) + record: count:up1 + - expr: count without(instance, pod, node) (up == 0) + record: count:up0 +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml new file mode 100755 index 000000000..1ff73f212 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml @@ -0,0 +1,39 @@ +{{- /* +Generated from 'kube-prometheus-node-recording.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusNodeRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-node-recording.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-node-recording.rules + rules: + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[3m])) BY (instance) + record: instance:node_cpu:rate:sum + - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance) + record: instance:node_network_receive_bytes:rate:sum + - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance) + record: instance:node_network_transmit_bytes:rate:sum + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance) + record: instance:node_cpu:ratio + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[5m])) + record: cluster:node_cpu:sum_rate5m + - expr: cluster:node_cpu_seconds_total:rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu)) + record: cluster:node_cpu:ratio +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml new file mode 100755 index 000000000..54de1ac33 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml @@ -0,0 +1,65 @@ +{{- /* +Generated from 'kube-scheduler.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeScheduler }} +{{- if (include "exporter.kubeScheduler.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-scheduler.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-scheduler.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.99' + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.9' + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_binding_duration_seconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) + labels: + quantile: '0.5' + record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-state-metrics.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-state-metrics.yaml new file mode 100755 index 000000000..207a61907 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kube-state-metrics.yaml @@ -0,0 +1,51 @@ +{{- /* +Generated from 'kube-state-metrics' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeStateMetrics.enabled .Values.defaultRules.rules.kubeStateMetrics }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-state-metrics" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-state-metrics + rules: + - alert: KubeStateMetricsListErrors + annotations: + message: kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatemetricslisterrors + expr: |- + (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m])) + / + sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m]))) + > 0.01 + for: 15m + labels: + severity: critical + - alert: KubeStateMetricsWatchErrors + annotations: + message: kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatemetricswatcherrors + expr: |- + (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m])) + / + sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics"}[5m]))) + > 0.01 + for: 15m + labels: + severity: critical +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubelet.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubelet.rules.yaml new file mode 100755 index 000000000..f9bd1ccd6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubelet.rules.yaml @@ -0,0 +1,39 @@ +{{- /* +Generated from 'kubelet.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubelet.enabled .Values.defaultRules.rules.kubelet }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubelet.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubelet.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.99' + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.9' + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (instance, le) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) + labels: + quantile: '0.5' + record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-apps.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-apps.yaml new file mode 100755 index 000000000..42d691356 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-apps.yaml @@ -0,0 +1,210 @@ +{{- /* +Generated from 'kubernetes-apps' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeStateMetrics.enabled .Values.defaultRules.rules.kubernetesApps }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-apps" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-apps + rules: + - alert: KubePodCrashLooping + annotations: + message: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is restarting {{`{{`}} printf "%.2f" $value {{`}}`}} times / 5 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodcrashlooping + expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[5m]) * 60 * 5 > 0 + for: 15m + labels: + severity: warning + - alert: KubePodNotReady + annotations: + message: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} has been in a non-ready state for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodnotready + expr: |- + sum by (namespace, pod) ( + max by(namespace, pod) ( + kube_pod_status_phase{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}", phase=~"Pending|Unknown"} + ) * on(namespace, pod) group_left(owner_kind) topk by(namespace, pod) ( + 1, max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!="Job"}) + ) + ) > 0 + for: 15m + labels: + severity: warning + - alert: KubeDeploymentGenerationMismatch + annotations: + message: Deployment generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} does not match, this indicates that the Deployment has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedeploymentgenerationmismatch + expr: |- + kube_deployment_status_observed_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_deployment_metadata_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + for: 15m + labels: + severity: warning + - alert: KubeDeploymentReplicasMismatch + annotations: + message: Deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedeploymentreplicasmismatch + expr: |- + ( + kube_deployment_spec_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_deployment_status_replicas_available{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_deployment_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: 15m + labels: + severity: warning + - alert: KubeStatefulSetReplicasMismatch + annotations: + message: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetreplicasmismatch + expr: |- + ( + kube_statefulset_status_replicas_ready{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) and ( + changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: 15m + labels: + severity: warning + - alert: KubeStatefulSetGenerationMismatch + annotations: + message: StatefulSet generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetgenerationmismatch + expr: |- + kube_statefulset_status_observed_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_metadata_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + for: 15m + labels: + severity: warning + - alert: KubeStatefulSetUpdateNotRolledOut + annotations: + message: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} update has not been rolled out. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetupdatenotrolledout + expr: |- + ( + max without (revision) ( + kube_statefulset_status_current_revision{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + unless + kube_statefulset_status_update_revision{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) + * + ( + kube_statefulset_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) + ) and ( + changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[5m]) + == + 0 + ) + for: 15m + labels: + severity: warning + - alert: KubeDaemonSetRolloutStuck + annotations: + message: Only {{`{{`}} $value | humanizePercentage {{`}}`}} of the desired Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are scheduled and ready. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetrolloutstuck + expr: |- + kube_daemonset_status_number_ready{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + / + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} < 1.00 + for: 15m + labels: + severity: warning + - alert: KubeContainerWaiting + annotations: + message: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} container {{`{{`}} $labels.container{{`}}`}} has been in waiting state for longer than 1 hour. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecontainerwaiting + expr: sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}) > 0 + for: 1h + labels: + severity: warning + - alert: KubeDaemonSetNotScheduled + annotations: + message: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are not scheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetnotscheduled + expr: |- + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + - + kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0 + for: 10m + labels: + severity: warning + - alert: KubeDaemonSetMisScheduled + annotations: + message: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are running where they are not supposed to run.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetmisscheduled + expr: kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0 + for: 15m + labels: + severity: warning + - alert: KubeJobCompletion + annotations: + message: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} is taking more than 12 hours to complete. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubejobcompletion + expr: kube_job_spec_completions{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - kube_job_status_succeeded{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0 + for: 12h + labels: + severity: warning + - alert: KubeJobFailed + annotations: + message: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} failed to complete. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubejobfailed + expr: kube_job_failed{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0 + for: 15m + labels: + severity: warning + - alert: KubeHpaReplicasMismatch + annotations: + message: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.hpa {{`}}`}} has not matched the desired number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubehpareplicasmismatch + expr: |- + (kube_hpa_status_desired_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_hpa_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}) + and + changes(kube_hpa_status_current_replicas[15m]) == 0 + for: 15m + labels: + severity: warning + - alert: KubeHpaMaxedOut + annotations: + message: HPA {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.hpa {{`}}`}} has been running at max replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubehpamaxedout + expr: |- + kube_hpa_status_current_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + == + kube_hpa_spec_max_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + for: 15m + labels: + severity: warning +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-resources.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-resources.yaml new file mode 100755 index 000000000..40596fef4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-resources.yaml @@ -0,0 +1,103 @@ +{{- /* +Generated from 'kubernetes-resources' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesResources }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-resources" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-resources + rules: + - alert: KubeCPUOvercommit + annotations: + message: Cluster has overcommitted CPU resource requests for Pods and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecpuovercommit + expr: |- + sum(namespace:kube_pod_container_resource_requests_cpu_cores:sum{}) + / + sum(kube_node_status_allocatable_cpu_cores) + > + (count(kube_node_status_allocatable_cpu_cores)-1) / count(kube_node_status_allocatable_cpu_cores) + for: 5m + labels: + severity: warning + - alert: KubeMemoryOvercommit + annotations: + message: Cluster has overcommitted memory resource requests for Pods and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubememoryovercommit + expr: |- + sum(namespace:kube_pod_container_resource_requests_memory_bytes:sum{}) + / + sum(kube_node_status_allocatable_memory_bytes) + > + (count(kube_node_status_allocatable_memory_bytes)-1) + / + count(kube_node_status_allocatable_memory_bytes) + for: 5m + labels: + severity: warning + - alert: KubeCPUQuotaOvercommit + annotations: + message: Cluster has overcommitted CPU resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecpuquotaovercommit + expr: |- + sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="cpu"}) + / + sum(kube_node_status_allocatable_cpu_cores) + > 1.5 + for: 5m + labels: + severity: warning + - alert: KubeMemoryQuotaOvercommit + annotations: + message: Cluster has overcommitted memory resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubememoryquotaovercommit + expr: |- + sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="memory"}) + / + sum(kube_node_status_allocatable_memory_bytes{job="node-exporter"}) + > 1.5 + for: 5m + labels: + severity: warning + - alert: KubeQuotaFullyUsed + annotations: + message: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} $value | humanizePercentage {{`}}`}} of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubequotafullyused + expr: |- + kube_resourcequota{job="kube-state-metrics", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0) + >= 1 + for: 15m + labels: + severity: info + - alert: CPUThrottlingHigh + annotations: + message: '{{`{{`}} $value | humanizePercentage {{`}}`}} throttling of CPU in namespace {{`{{`}} $labels.namespace {{`}}`}} for container {{`{{`}} $labels.container {{`}}`}} in pod {{`{{`}} $labels.pod {{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-cputhrottlinghigh + expr: |- + sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (container, pod, namespace) + / + sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace) + > ( 25 / 100 ) + for: 15m + labels: + severity: info +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-storage.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-storage.yaml new file mode 100755 index 000000000..52092f0ad --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-storage.yaml @@ -0,0 +1,63 @@ +{{- /* +Generated from 'kubernetes-storage' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesStorage }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-storage" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-storage + rules: + - alert: KubePersistentVolumeFillingUp + annotations: + message: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is only {{`{{`}} $value | humanizePercentage {{`}}`}} free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumefillingup + expr: |- + kubelet_volume_stats_available_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + < 0.03 + for: 1m + labels: + severity: critical + - alert: KubePersistentVolumeFillingUp + annotations: + message: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is expected to fill up within four days. Currently {{`{{`}} $value | humanizePercentage {{`}}`}} is available. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumefillingup + expr: |- + ( + kubelet_volume_stats_available_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"} + ) < 0.15 + and + predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + for: 1h + labels: + severity: warning + - alert: KubePersistentVolumeErrors + annotations: + message: The persistent volume {{`{{`}} $labels.persistentvolume {{`}}`}} has status {{`{{`}} $labels.phase {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumeerrors + expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0 + for: 5m + labels: + severity: critical +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml new file mode 100755 index 000000000..dde1f7afb --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml @@ -0,0 +1,66 @@ +{{- /* +Generated from 'kubernetes-system-apiserver' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-apiserver" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-apiserver + rules: + - alert: KubeClientCertificateExpiration + annotations: + message: A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclientcertificateexpiration + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 + labels: + severity: warning + - alert: KubeClientCertificateExpiration + annotations: + message: A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclientcertificateexpiration + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 + labels: + severity: critical + - alert: AggregatedAPIErrors + annotations: + message: An aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has reported errors. The number of errors have increased for it in the past five minutes. High values indicate that the availability of the service changes too often. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-aggregatedapierrors + expr: sum by(name, namespace)(increase(aggregator_unavailable_apiservice_count[5m])) > 2 + labels: + severity: warning + - alert: AggregatedAPIDown + annotations: + message: An aggregated API {{`{{`}} $labels.name {{`}}`}}/{{`{{`}} $labels.namespace {{`}}`}} has been only {{`{{`}} $value | humanize {{`}}`}}% available over the last 5m. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-aggregatedapidown + expr: (1 - max by(name, namespace)(avg_over_time(aggregator_unavailable_apiservice[5m]))) * 100 < 90 + for: 5m + labels: + severity: warning +{{- if .Values.kubeApiServer.enabled }} + - alert: KubeAPIDown + annotations: + message: KubeAPI has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapidown + expr: absent(up{job="apiserver"} == 1) + for: 15m + labels: + severity: critical +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml new file mode 100755 index 000000000..73292bedd --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml @@ -0,0 +1,39 @@ +{{- /* +Generated from 'kubernetes-system-controller-manager' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-controller-manager" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-controller-manager + rules: +{{- if (include "exporter.kubeControllerManager.enabled" .)}} + - alert: KubeControllerManagerDown + annotations: + message: KubeControllerManager has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecontrollermanagerdown + expr: absent(up{job="{{ include "exporter.kubeControllerManager.jobName" . }}"} == 1) + for: 15m + labels: + severity: critical +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml new file mode 100755 index 000000000..6d75c6f55 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml @@ -0,0 +1,91 @@ +{{- /* +Generated from 'kubernetes-system-kubelet' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kubelet" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-kubelet + rules: + - alert: KubeNodeNotReady + annotations: + message: '{{`{{`}} $labels.node {{`}}`}} has been unready for more than 15 minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubenodenotready + expr: kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0 + for: 15m + labels: + severity: warning + - alert: KubeNodeUnreachable + annotations: + message: '{{`{{`}} $labels.node {{`}}`}} is unreachable and some workloads may be rescheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubenodeunreachable + expr: (kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1 + labels: + severity: warning + - alert: KubeletTooManyPods + annotations: + message: Kubelet '{{`{{`}} $labels.node {{`}}`}}' is running at {{`{{`}} $value | humanizePercentage {{`}}`}} of its Pod capacity. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubelettoomanypods + expr: |- + count by(node) ( + (kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job="kube-state-metrics"}) + ) + / + max by(node) ( + kube_node_status_capacity_pods{job="kube-state-metrics"} != 1 + ) > 0.95 + for: 15m + labels: + severity: warning + - alert: KubeNodeReadinessFlapping + annotations: + message: The readiness status of node {{`{{`}} $labels.node {{`}}`}} has changed {{`{{`}} $value {{`}}`}} times in the last 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubenodereadinessflapping + expr: sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (node) > 2 + for: 15m + labels: + severity: warning + - alert: KubeletPlegDurationHigh + annotations: + message: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletplegdurationhigh + expr: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10 + for: 5m + labels: + severity: warning + - alert: KubeletPodStartUpLatencyHigh + annotations: + message: Kubelet Pod startup 99th percentile latency is {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletpodstartuplatencyhigh + expr: histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"} > 60 + for: 15m + labels: + severity: warning +{{- if .Values.prometheusOperator.kubeletService.enabled }} + - alert: KubeletDown + annotations: + message: Kubelet has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletdown + expr: absent(up{job="kubelet", metrics_path="/metrics"} == 1) + for: 15m + labels: + severity: critical +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml new file mode 100755 index 000000000..9a2eb8a9b --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml @@ -0,0 +1,39 @@ +{{- /* +Generated from 'kubernetes-system-scheduler' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeScheduler }} +{{- if (include "exporter.kubeScheduler.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-scheduler" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system-scheduler + rules: +{{- if (include "exporter.kubeScheduler.enabled" .)}} + - alert: KubeSchedulerDown + annotations: + message: KubeScheduler has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeschedulerdown + expr: absent(up{job="{{ include "exporter.kubeScheduler.jobName" . }}"} == 1) + for: 15m + labels: + severity: critical +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system.yaml new file mode 100755 index 000000000..8fd0ad318 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/kubernetes-system.yaml @@ -0,0 +1,47 @@ +{{- /* +Generated from 'kubernetes-system' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system + rules: + - alert: KubeVersionMismatch + annotations: + message: There are {{`{{`}} $value {{`}}`}} different semantic versions of Kubernetes components running. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeversionmismatch + expr: count(count by (gitVersion) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"gitVersion","$1","gitVersion","(v[0-9]*.[0-9]*).*"))) > 1 + for: 15m + labels: + severity: warning + - alert: KubeClientErrors + annotations: + message: Kubernetes API server client '{{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.instance {{`}}`}}' is experiencing {{`{{`}} $value | humanizePercentage {{`}}`}} errors.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclienterrors + expr: |- + (sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (instance, job) + / + sum(rate(rest_client_requests_total[5m])) by (instance, job)) + > 0.01 + for: 15m + labels: + severity: warning +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/node-exporter.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/node-exporter.rules.yaml new file mode 100755 index 000000000..845eb0c90 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/node-exporter.rules.yaml @@ -0,0 +1,79 @@ +{{- /* +Generated from 'node-exporter.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.nodeExporter.enabled .Values.defaultRules.rules.node }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter.rules + rules: + - expr: |- + count without (cpu) ( + count without (mode) ( + node_cpu_seconds_total{job="node-exporter"} + ) + ) + record: instance:node_num_cpu:sum + - expr: |- + 1 - avg without (cpu, mode) ( + rate(node_cpu_seconds_total{job="node-exporter", mode="idle"}[1m]) + ) + record: instance:node_cpu_utilisation:rate1m + - expr: |- + ( + node_load1{job="node-exporter"} + / + instance:node_num_cpu:sum{job="node-exporter"} + ) + record: instance:node_load1_per_cpu:ratio + - expr: |- + 1 - ( + node_memory_MemAvailable_bytes{job="node-exporter"} + / + node_memory_MemTotal_bytes{job="node-exporter"} + ) + record: instance:node_memory_utilisation:ratio + - expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[1m]) + record: instance:node_vmstat_pgmajfault:rate1m + - expr: rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"}[1m]) + record: instance_device:node_disk_io_time_seconds:rate1m + - expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"}[1m]) + record: instance_device:node_disk_io_time_weighted_seconds:rate1m + - expr: |- + sum without (device) ( + rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[1m]) + ) + record: instance:node_network_receive_bytes_excluding_lo:rate1m + - expr: |- + sum without (device) ( + rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[1m]) + ) + record: instance:node_network_transmit_bytes_excluding_lo:rate1m + - expr: |- + sum without (device) ( + rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[1m]) + ) + record: instance:node_network_receive_drop_excluding_lo:rate1m + - expr: |- + sum without (device) ( + rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[1m]) + ) + record: instance:node_network_transmit_drop_excluding_lo:rate1m +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/node-exporter.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/node-exporter.yaml new file mode 100755 index 000000000..17e112b65 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/node-exporter.yaml @@ -0,0 +1,210 @@ +{{- /* +Generated from 'node-exporter' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.nodeExporter.enabled .Values.defaultRules.rules.node }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-exporter + rules: + - alert: NodeFilesystemSpaceFillingUp + annotations: + description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 24 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 40 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 + ) + for: 1h + labels: + severity: warning + - alert: NodeFilesystemSpaceFillingUp + annotations: + description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemspacefillingup + summary: Filesystem is predicted to run out of space within the next 4 hours. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 15 + and + predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 + ) + for: 1h + labels: + severity: critical + - alert: NodeFilesystemAlmostOutOfSpace + annotations: + description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutofspace + summary: Filesystem has less than 5% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 + ) + for: 1h + labels: + severity: warning + - alert: NodeFilesystemAlmostOutOfSpace + annotations: + description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutofspace + summary: Filesystem has less than 3% space left. + expr: |- + ( + node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 + ) + for: 1h + labels: + severity: critical + - alert: NodeFilesystemFilesFillingUp + annotations: + description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 24 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 40 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!=""}[6h], 24*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 + ) + for: 1h + labels: + severity: warning + - alert: NodeFilesystemFilesFillingUp + annotations: + description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemfilesfillingup + summary: Filesystem is predicted to run out of inodes within the next 4 hours. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 20 + and + predict_linear(node_filesystem_files_free{job="node-exporter",fstype!=""}[6h], 4*60*60) < 0 + and + node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 + ) + for: 1h + labels: + severity: critical + - alert: NodeFilesystemAlmostOutOfFiles + annotations: + description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutoffiles + summary: Filesystem has less than 5% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 5 + and + node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 + ) + for: 1h + labels: + severity: warning + - alert: NodeFilesystemAlmostOutOfFiles + annotations: + description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutoffiles + summary: Filesystem has less than 3% inodes left. + expr: |- + ( + node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 3 + and + node_filesystem_readonly{job="node-exporter",fstype!=""} == 0 + ) + for: 1h + labels: + severity: critical + - alert: NodeNetworkReceiveErrs + annotations: + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodenetworkreceiveerrs + summary: Network interface is reporting many receive errors. + expr: increase(node_network_receive_errs_total[2m]) > 10 + for: 1h + labels: + severity: warning + - alert: NodeNetworkTransmitErrs + annotations: + description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodenetworktransmiterrs + summary: Network interface is reporting many transmit errors. + expr: increase(node_network_transmit_errs_total[2m]) > 10 + for: 1h + labels: + severity: warning + - alert: NodeHighNumberConntrackEntriesUsed + annotations: + description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodehighnumberconntrackentriesused + summary: Number of conntrack are getting close to the limit. + expr: (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75 + labels: + severity: warning + - alert: NodeTextFileCollectorScrapeError + annotations: + description: Node Exporter text file collector failed to scrape. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodetextfilecollectorscrapeerror + summary: Node Exporter text file collector failed to scrape. + expr: node_textfile_scrape_error{job="node-exporter"} == 1 + labels: + severity: warning + - alert: NodeClockSkewDetected + annotations: + message: Clock on {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 300s. Ensure NTP is configured correctly on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodeclockskewdetected + summary: Clock skew detected. + expr: |- + ( + node_timex_offset_seconds > 0.05 + and + deriv(node_timex_offset_seconds[5m]) >= 0 + ) + or + ( + node_timex_offset_seconds < -0.05 + and + deriv(node_timex_offset_seconds[5m]) <= 0 + ) + for: 10m + labels: + severity: warning + - alert: NodeClockNotSynchronising + annotations: + message: Clock on {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodeclocknotsynchronising + summary: Clock not synchronising. + expr: min_over_time(node_timex_sync_status[5m]) == 0 + for: 10m + labels: + severity: warning +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/node-network.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/node-network.yaml new file mode 100755 index 000000000..b6f4eda74 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/node-network.yaml @@ -0,0 +1,34 @@ +{{- /* +Generated from 'node-network' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.network }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-network" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-network + rules: + - alert: NodeNetworkInterfaceFlapping + annotations: + message: Network interface "{{`{{`}} $labels.device {{`}}`}}" changing it's up status often on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}}" + expr: changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2 + for: 2m + labels: + severity: warning +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/node.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/node.rules.yaml new file mode 100755 index 000000000..33af72c2b --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/node.rules.yaml @@ -0,0 +1,53 @@ +{{- /* +Generated from 'node.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.nodeExporter.enabled .Values.defaultRules.rules.node }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node.rules + rules: + - expr: sum(min(kube_pod_info{node!=""}) by (cluster, node)) + record: ':kube_pod_info_node_count:' + - expr: |- + topk by(namespace, pod) (1, + max by (node, namespace, pod) ( + label_replace(kube_pod_info{job="kube-state-metrics",node!=""}, "pod", "$1", "pod", "(.*)") + )) + record: 'node_namespace_pod:kube_pod_info:' + - expr: |- + count by (cluster, node) (sum by (node, cpu) ( + node_cpu_seconds_total{job="node-exporter"} + * on (namespace, pod) group_left(node) + node_namespace_pod:kube_pod_info: + )) + record: node:node_num_cpu:sum + - expr: |- + sum( + node_memory_MemAvailable_bytes{job="node-exporter"} or + ( + node_memory_Buffers_bytes{job="node-exporter"} + + node_memory_Cached_bytes{job="node-exporter"} + + node_memory_MemFree_bytes{job="node-exporter"} + + node_memory_Slab_bytes{job="node-exporter"} + ) + ) by (cluster) + record: :node_memory_MemAvailable_bytes:sum +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/prometheus-operator.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/prometheus-operator.yaml new file mode 100755 index 000000000..c420c3ca6 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/prometheus-operator.yaml @@ -0,0 +1,57 @@ +{{- /* +Generated from 'prometheus-operator' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheusOperator }} +{{- $operatorJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "operator" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus-operator + rules: + - alert: PrometheusOperatorListErrors + annotations: + message: Errors while performing List operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + expr: (sum by (controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_list_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[1h]))) > 0.4 + for: 15m + labels: + severity: warning + - alert: PrometheusOperatorWatchErrors + annotations: + message: Errors while performing Watch operations in controller {{`{{`}}$labels.controller{{`}}`}} in {{`{{`}}$labels.namespace{{`}}`}} namespace. + expr: (sum by (controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[1h])) / sum by (controller,namespace) (rate(prometheus_operator_watch_operations_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[1h]))) > 0.4 + for: 15m + labels: + severity: warning + - alert: PrometheusOperatorReconcileErrors + annotations: + message: Errors while reconciling {{`{{`}} $labels.controller {{`}}`}} in {{`{{`}} $labels.namespace {{`}}`}} Namespace. + expr: rate(prometheus_operator_reconcile_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0.1 + for: 10m + labels: + severity: warning + - alert: PrometheusOperatorNodeLookupErrors + annotations: + message: Errors while reconciling Prometheus in {{`{{`}} $labels.namespace {{`}}`}} Namespace. + expr: rate(prometheus_operator_node_address_lookup_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0.1 + for: 10m + labels: + severity: warning +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/prometheus.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/prometheus.yaml new file mode 100755 index 000000000..66976c4b3 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules-1.14/prometheus.yaml @@ -0,0 +1,202 @@ +{{- /* +Generated from 'prometheus' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheus }} +{{- $prometheusJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus + rules: + - alert: PrometheusBadConfig + annotations: + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to reload its configuration. + summary: Failed Prometheus configuration reload. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_config_last_reload_successful{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) == 0 + for: 10m + labels: + severity: critical + - alert: PrometheusNotificationQueueRunningFull + annotations: + description: Alert notification queue of Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is running full. + summary: Prometheus alert notification queue predicted to run full in less than 30m. + expr: |- + # Without min_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + predict_linear(prometheus_notifications_queue_length{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m], 60 * 30) + > + min_over_time(prometheus_notifications_queue_capacity{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: 15m + labels: + severity: warning + - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers + annotations: + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to Alertmanager {{`{{`}}$labels.alertmanager{{`}}`}}.' + summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager. + expr: |- + ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + * 100 + > 1 + for: 15m + labels: + severity: warning + - alert: PrometheusErrorSendingAlertsToAnyAlertmanager + annotations: + description: '{{`{{`}} printf "%.1f" $value {{`}}`}}% minimum errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} to any Alertmanager.' + summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager. + expr: |- + min without(alertmanager) ( + rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + / + rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + * 100 + > 3 + for: 15m + labels: + severity: critical + - alert: PrometheusNotConnectedToAlertmanagers + annotations: + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not connected to any Alertmanagers. + summary: Prometheus is not connected to any Alertmanagers. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + max_over_time(prometheus_notifications_alertmanagers_discovered{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) < 1 + for: 10m + labels: + severity: warning + - alert: PrometheusTSDBReloadsFailing + annotations: + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} reload failures over the last 3h. + summary: Prometheus has issues reloading blocks from disk. + expr: increase(prometheus_tsdb_reloads_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: 4h + labels: + severity: warning + - alert: PrometheusTSDBCompactionsFailing + annotations: + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has detected {{`{{`}}$value | humanize{{`}}`}} compaction failures over the last 3h. + summary: Prometheus has issues compacting blocks. + expr: increase(prometheus_tsdb_compactions_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[3h]) > 0 + for: 4h + labels: + severity: warning + - alert: PrometheusNotIngestingSamples + annotations: + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is not ingesting samples. + summary: Prometheus is not ingesting samples. + expr: rate(prometheus_tsdb_head_samples_appended_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) <= 0 + for: 10m + labels: + severity: warning + - alert: PrometheusDuplicateTimestamps + annotations: + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with different values but duplicated timestamp. + summary: Prometheus is dropping samples with duplicate timestamps. + expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 10m + labels: + severity: warning + - alert: PrometheusOutOfOrderTimestamps + annotations: + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} is dropping {{`{{`}} printf "%.4g" $value {{`}}`}} samples/s with timestamps arriving out of order. + summary: Prometheus drops samples with out-of-order timestamps. + expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 10m + labels: + severity: warning + - alert: PrometheusRemoteStorageFailures + annotations: + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} failed to send {{`{{`}} printf "%.1f" $value {{`}}`}}% of the samples to {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}} + summary: Prometheus fails to send samples to remote storage. + expr: |- + ( + rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + / + ( + rate(prometheus_remote_storage_failed_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + + + rate(prometheus_remote_storage_succeeded_samples_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + ) + * 100 + > 1 + for: 15m + labels: + severity: critical + - alert: PrometheusRemoteWriteBehind + annotations: + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write is {{`{{`}} printf "%.1f" $value {{`}}`}}s behind for {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}. + summary: Prometheus remote write is behind. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + - on(job, instance) group_right + max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + > 120 + for: 15m + labels: + severity: critical + - alert: PrometheusRemoteWriteDesiredShards + annotations: + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} remote write desired shards calculation wants to run {{`{{`}} $value {{`}}`}} shards for queue {{`{{`}} $labels.remote_name{{`}}`}}:{{`{{`}} $labels.url {{`}}`}}, which is more than the max of {{`{{`}} printf `prometheus_remote_storage_shards_max{instance="%s",job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}` $labels.instance | query | first | value {{`}}`}}. + summary: Prometheus remote write desired shards calculation wants to run more than configured max shards. + expr: |- + # Without max_over_time, failed scrapes could create false negatives, see + # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. + ( + max_over_time(prometheus_remote_storage_shards_desired{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + > + max_over_time(prometheus_remote_storage_shards_max{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) + ) + for: 15m + labels: + severity: warning + - alert: PrometheusRuleFailures + annotations: + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has failed to evaluate {{`{{`}} printf "%.0f" $value {{`}}`}} rules in the last 5m. + summary: Prometheus is failing rule evaluations. + expr: increase(prometheus_rule_evaluation_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 15m + labels: + severity: critical + - alert: PrometheusMissingRuleEvaluations + annotations: + description: Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has missed {{`{{`}} printf "%.0f" $value {{`}}`}} rule group evaluations in the last 5m. + summary: Prometheus is missing rule evaluations due to slow rule group evaluation. + expr: increase(prometheus_rule_group_iterations_missed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 15m + labels: + severity: warning +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/alertmanager.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/alertmanager.rules.yaml new file mode 100755 index 000000000..cdc814d27 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/alertmanager.rules.yaml @@ -0,0 +1,54 @@ +{{- /* +Generated from 'alertmanager.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.alertmanager }} +{{- $operatorJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "operator" }} +{{- $alertmanagerJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: alertmanager.rules + rules: + - alert: AlertmanagerConfigInconsistent + annotations: + message: The configuration of the instances of the Alertmanager cluster `{{`{{`}}$labels.service{{`}}`}}` are out of sync. + expr: count_values("config_hash", alertmanager_config_hash{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}) BY (service) / ON(service) GROUP_LEFT() label_replace(max(prometheus_operator_spec_replicas{job="{{ $operatorJob }}",namespace="{{ $namespace }}",controller="alertmanager"}) by (name, job, namespace, controller), "service", "$1", "name", "(.*)") != 1 + for: 5m + labels: + severity: critical + - alert: AlertmanagerFailedReload + annotations: + message: Reloading Alertmanager's configuration has failed for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}}. + expr: alertmanager_config_last_reload_successful{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} == 0 + for: 10m + labels: + severity: warning + - alert: AlertmanagerMembersInconsistent + annotations: + message: Alertmanager has not found all other members of the cluster. + expr: |- + alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} + != on (service) GROUP_LEFT() + count by (service) (alertmanager_cluster_members{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"}) + for: 5m + labels: + severity: critical +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/etcd.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/etcd.yaml new file mode 100755 index 000000000..e60a1c1cf --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/etcd.yaml @@ -0,0 +1,157 @@ +{{- /* +Generated from 'etcd' group from https://raw.githubusercontent.com/etcd-io/etcd/master/Documentation/op-guide/etcd3_alert.rules.yml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.etcd }} +{{- if (include "exporter.kubeEtcd.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "etcd" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: etcd + rules: + - alert: etcdMembersDown + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": members are down ({{`{{`}} $value {{`}}`}}).' + expr: |- + max by (job) ( + sum by (job) (up{job=~".*etcd.*"} == bool 0) + or + count by (job,endpoint) ( + sum by (job,endpoint,To) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[3m])) > 0.01 + ) + ) + > 0 + for: 3m + labels: + severity: critical + - alert: etcdInsufficientMembers + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": insufficient members ({{`{{`}} $value {{`}}`}}).' + expr: sum(up{job=~".*etcd.*"} == bool 1) by (job) < ((count(up{job=~".*etcd.*"}) by (job) + 1) / 2) + for: 3m + labels: + severity: critical + - alert: etcdNoLeader + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member {{`{{`}} $labels.instance {{`}}`}} has no leader.' + expr: etcd_server_has_leader{job=~".*etcd.*"} == 0 + for: 1m + labels: + severity: critical + - alert: etcdHighNumberOfLeaderChanges + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.' + expr: increase((max by (job) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 3 + for: 5m + labels: + severity: warning + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method) + > 1 + for: 10m + labels: + severity: warning + - alert: etcdHighNumberOfFailedGRPCRequests + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.grpc_method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + expr: |- + 100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code!="OK"}[5m])) BY (job, instance, grpc_service, grpc_method) + / + sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) BY (job, instance, grpc_service, grpc_method) + > 5 + for: 5m + labels: + severity: critical + - alert: etcdGRPCRequestsSlow + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": gRPC requests to {{`{{`}} $labels.grpc_method {{`}}`}} are taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + expr: |- + histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le)) + > 0.15 + for: 10m + labels: + severity: critical + - alert: etcdMemberCommunicationSlow + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member communication with {{`{{`}} $labels.To {{`}}`}} is taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + expr: |- + histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.15 + for: 10m + labels: + severity: warning + - alert: etcdHighNumberOfFailedProposals + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last 30 minutes on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5 + for: 15m + labels: + severity: warning + - alert: etcdHighFsyncDurations + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + expr: |- + histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.5 + for: 10m + labels: + severity: warning + - alert: etcdHighCommitDurations + annotations: + message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile commit durations {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + expr: |- + histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) + > 0.25 + for: 10m + labels: + severity: warning + - alert: etcdHighNumberOfFailedHTTPRequests + annotations: + message: '{{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}' + expr: |- + sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) + BY (method) > 0.01 + for: 10m + labels: + severity: warning + - alert: etcdHighNumberOfFailedHTTPRequests + annotations: + message: '{{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.' + expr: |- + sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m])) + BY (method) > 0.05 + for: 10m + labels: + severity: critical + - alert: etcdHTTPRequestsSlow + annotations: + message: etcd instance {{`{{`}} $labels.instance {{`}}`}} HTTP requests to {{`{{`}} $labels.method {{`}}`}} are slow. + expr: |- + histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m])) + > 0.15 + for: 10m + labels: + severity: warning +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/general.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/general.rules.yaml new file mode 100755 index 000000000..cc5f07093 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/general.rules.yaml @@ -0,0 +1,50 @@ +{{- /* +Generated from 'general.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.general }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "general.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: general.rules + rules: + - alert: TargetDown + annotations: + message: '{{`{{`}} $value {{`}}`}}% of the {{`{{`}} $labels.job {{`}}`}} targets are down.' + expr: 100 * (count(up == 0) BY (job) / count(up) BY (job)) > 10 + for: 10m + labels: + severity: warning + - alert: Watchdog + annotations: + message: 'This is an alert meant to ensure that the entire alerting pipeline is functional. + + This alert is always firing, therefore it should always be firing in Alertmanager + + and always fire against a receiver. There are integrations with various notification + + mechanisms that send a notification when this alert is not firing. For example the + + "DeadMansSnitch" integration in PagerDuty. + + ' + expr: vector(1) + labels: + severity: none +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/k8s.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/k8s.rules.yaml new file mode 100755 index 000000000..08aa7fe2b --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/k8s.rules.yaml @@ -0,0 +1,83 @@ +{{- /* +Generated from 'k8s.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.k8s }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "k8s.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: k8s.rules + rules: + - expr: sum(rate(container_cpu_usage_seconds_total{job="kubelet", image!="", container_name!=""}[5m])) by (namespace) + record: namespace:container_cpu_usage_seconds_total:sum_rate + - expr: sum(container_memory_usage_bytes{job="kubelet", image!="", container_name!=""}) by (namespace) + record: namespace:container_memory_usage_bytes:sum + - expr: |- + sum by (namespace, pod_name, container_name) ( + rate(container_cpu_usage_seconds_total{job="kubelet", image!="", container_name!=""}[5m]) + ) + record: namespace_pod_name_container_name:container_cpu_usage_seconds_total:sum_rate + - expr: |- + sum by(namespace) ( + kube_pod_container_resource_requests_memory_bytes{job="kube-state-metrics"} + * on (endpoint, instance, job, namespace, pod, service) + group_left(phase) (kube_pod_status_phase{phase=~"^(Pending|Running)$"} == 1) + ) + record: namespace_name:kube_pod_container_resource_requests_memory_bytes:sum + - expr: |- + sum by (namespace) ( + kube_pod_container_resource_requests_cpu_cores{job="kube-state-metrics"} + * on (endpoint, instance, job, namespace, pod, service) + group_left(phase) (kube_pod_status_phase{phase=~"^(Pending|Running)$"} == 1) + ) + record: namespace_name:kube_pod_container_resource_requests_cpu_cores:sum + - expr: |- + sum( + label_replace( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"}, + "replicaset", "$1", "owner_name", "(.*)" + ) * on(replicaset, namespace) group_left(owner_name) kube_replicaset_owner{job="kube-state-metrics"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) by (namespace, workload, pod) + labels: + workload_type: deployment + record: mixin_pod_workload + - expr: |- + sum( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) by (namespace, workload, pod) + labels: + workload_type: daemonset + record: mixin_pod_workload + - expr: |- + sum( + label_replace( + kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"}, + "workload", "$1", "owner_name", "(.*)" + ) + ) by (namespace, workload, pod) + labels: + workload_type: statefulset + record: mixin_pod_workload +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kube-apiserver.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kube-apiserver.rules.yaml new file mode 100755 index 000000000..e3a929692 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kube-apiserver.rules.yaml @@ -0,0 +1,39 @@ +{{- /* +Generated from 'kube-apiserver.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserver }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-apiserver.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-apiserver.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(apiserver_request_latencies_bucket{job="apiserver"}[5m])) without(instance, pod)) / 1e+06 + labels: + quantile: '0.99' + record: cluster_quantile:apiserver_request_latencies:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(apiserver_request_latencies_bucket{job="apiserver"}[5m])) without(instance, pod)) / 1e+06 + labels: + quantile: '0.9' + record: cluster_quantile:apiserver_request_latencies:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(apiserver_request_latencies_bucket{job="apiserver"}[5m])) without(instance, pod)) / 1e+06 + labels: + quantile: '0.5' + record: cluster_quantile:apiserver_request_latencies:histogram_quantile +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kube-prometheus-node-alerting.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kube-prometheus-node-alerting.rules.yaml new file mode 100755 index 000000000..cbd8ea39c --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kube-prometheus-node-alerting.rules.yaml @@ -0,0 +1,41 @@ +{{- /* +Generated from 'kube-prometheus-node-alerting.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusNodeAlerting }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-node-alerting.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-node-alerting.rules + rules: + - alert: NodeDiskRunningFull + annotations: + message: Device {{`{{`}} $labels.device {{`}}`}} of node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} will be full within the next 24 hours. + expr: '(node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[6h], 3600 * 24) < 0)' + for: 30m + labels: + severity: warning + - alert: NodeDiskRunningFull + annotations: + message: Device {{`{{`}} $labels.device {{`}}`}} of node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} will be full within the next 2 hours. + expr: '(node:node_filesystem_usage: > 0.85) and (predict_linear(node:node_filesystem_avail:[30m], 3600 * 2) < 0)' + for: 10m + labels: + severity: critical +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kube-prometheus-node-recording.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kube-prometheus-node-recording.rules.yaml new file mode 100755 index 000000000..87f072fd0 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kube-prometheus-node-recording.rules.yaml @@ -0,0 +1,41 @@ +{{- /* +Generated from 'kube-prometheus-node-recording.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubePrometheusNodeRecording }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus-node-recording.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-prometheus-node-recording.rules + rules: + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[3m])) BY (instance) + record: instance:node_cpu:rate:sum + - expr: sum((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"})) BY (instance) + record: instance:node_filesystem_usage:sum + - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance) + record: instance:node_network_receive_bytes:rate:sum + - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance) + record: instance:node_network_transmit_bytes:rate:sum + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance) + record: instance:node_cpu:ratio + - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[5m])) + record: cluster:node_cpu:sum_rate5m + - expr: cluster:node_cpu_seconds_total:rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu)) + record: cluster:node_cpu:ratio +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kube-scheduler.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kube-scheduler.rules.yaml new file mode 100755 index 000000000..e8de8ed6e --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kube-scheduler.rules.yaml @@ -0,0 +1,65 @@ +{{- /* +Generated from 'kube-scheduler.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubeScheduler }} +{{- if (include "exporter.kubeScheduler.enabled" .)}} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-scheduler.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kube-scheduler.rules + rules: + - expr: histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_latency_microseconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) / 1e+06 + labels: + quantile: '0.99' + record: cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_latency_microseconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) / 1e+06 + labels: + quantile: '0.99' + record: cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile + - expr: histogram_quantile(0.99, sum(rate(scheduler_binding_latency_microseconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) / 1e+06 + labels: + quantile: '0.99' + record: cluster_quantile:scheduler_binding_latency:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_latency_microseconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) / 1e+06 + labels: + quantile: '0.9' + record: cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_latency_microseconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) / 1e+06 + labels: + quantile: '0.9' + record: cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile + - expr: histogram_quantile(0.9, sum(rate(scheduler_binding_latency_microseconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) / 1e+06 + labels: + quantile: '0.9' + record: cluster_quantile:scheduler_binding_latency:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_latency_microseconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) / 1e+06 + labels: + quantile: '0.5' + record: cluster_quantile:scheduler_e2e_scheduling_latency:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_latency_microseconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) / 1e+06 + labels: + quantile: '0.5' + record: cluster_quantile:scheduler_scheduling_algorithm_latency:histogram_quantile + - expr: histogram_quantile(0.5, sum(rate(scheduler_binding_latency_microseconds_bucket{job="{{ include "exporter.kubeScheduler.jobName" . }}"}[5m])) without(instance, pod)) / 1e+06 + labels: + quantile: '0.5' + record: cluster_quantile:scheduler_binding_latency:histogram_quantile +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-absent.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-absent.yaml new file mode 100755 index 000000000..de8ebc7a0 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-absent.yaml @@ -0,0 +1,129 @@ +{{- /* +Generated from 'kubernetes-absent' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesAbsent }} +{{- $operatorJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "operator" }} +{{- $prometheusJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} +{{- $alertmanagerJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-absent" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-absent + rules: +{{- if .Values.alertmanager.enabled }} + - alert: AlertmanagerDown + annotations: + message: Alertmanager has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-alertmanagerdown + expr: absent(up{job="{{ $alertmanagerJob }}",namespace="{{ $namespace }}"} == 1) + for: 15m + labels: + severity: critical +{{- end }} +{{- if .Values.kubeDns.enabled }} + - alert: CoreDNSDown + annotations: + message: CoreDNS has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-corednsdown + expr: absent(up{job="kube-dns"} == 1) + for: 15m + labels: + severity: critical +{{- end }} +{{- if .Values.kubeApiServer.enabled }} + - alert: KubeAPIDown + annotations: + message: KubeAPI has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapidown + expr: absent(up{job="apiserver"} == 1) + for: 15m + labels: + severity: critical +{{- end }} +{{- if (include "exporter.kubeControllerManager.enabled" .)}} + - alert: KubeControllerManagerDown + annotations: + message: KubeControllerManager has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecontrollermanagerdown + expr: absent(up{job="{{ include "exporter.kubeControllerManager.jobName" . }}"} == 1) + for: 15m + labels: + severity: critical +{{- end }} +{{- if (include "exporter.kubeScheduler.enabled" .)}} + - alert: KubeSchedulerDown + annotations: + message: KubeScheduler has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeschedulerdown + expr: absent(up{job="{{ include "exporter.kubeScheduler.jobName" . }}"} == 1) + for: 15m + labels: + severity: critical +{{- end }} +{{- if .Values.kubeStateMetrics.enabled }} + - alert: KubeStateMetricsDown + annotations: + message: KubeStateMetrics has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatemetricsdown + expr: absent(up{job="kube-state-metrics"} == 1) + for: 15m + labels: + severity: critical +{{- end }} +{{- if .Values.prometheusOperator.kubeletService.enabled }} + - alert: KubeletDown + annotations: + message: Kubelet has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletdown + expr: absent(up{job="kubelet"} == 1) + for: 15m + labels: + severity: critical +{{- end }} +{{- if .Values.nodeExporter.enabled }} + - alert: NodeExporterDown + annotations: + message: NodeExporter has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-nodeexporterdown + expr: absent(up{job="node-exporter"} == 1) + for: 15m + labels: + severity: critical +{{- end }} + - alert: PrometheusDown + annotations: + message: Prometheus has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusdown + expr: absent(up{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"} == 1) + for: 15m + labels: + severity: critical +{{- if .Values.prometheusOperator.enabled }} + - alert: PrometheusOperatorDown + annotations: + message: PrometheusOperator has disappeared from Prometheus target discovery. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-prometheusoperatordown + expr: absent(up{job="{{ $operatorJob }}",namespace="{{ $namespace }}"} == 1) + for: 15m + labels: + severity: critical +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-apps.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-apps.yaml new file mode 100755 index 000000000..41d091c58 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-apps.yaml @@ -0,0 +1,161 @@ +{{- /* +Generated from 'kubernetes-apps' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeStateMetrics.enabled .Values.defaultRules.rules.kubernetesApps }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-apps" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-apps + rules: + - alert: KubePodCrashLooping + annotations: + message: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is restarting {{`{{`}} printf "%.2f" $value {{`}}`}} times / 5 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodcrashlooping + expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[15m]) * 60 * 5 > 0 + for: 1h + labels: + severity: critical + - alert: KubePodNotReady + annotations: + message: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} has been in a non-ready state for longer than an hour. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodnotready + expr: sum by (namespace, pod) (kube_pod_status_phase{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}", phase=~"Pending|Unknown"}) > 0 + for: 1h + labels: + severity: critical + - alert: KubeDeploymentGenerationMismatch + annotations: + message: Deployment generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} does not match, this indicates that the Deployment has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedeploymentgenerationmismatch + expr: |- + kube_deployment_status_observed_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_deployment_metadata_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + for: 15m + labels: + severity: critical + - alert: KubeDeploymentReplicasMismatch + annotations: + message: Deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} has not matched the expected number of replicas for longer than an hour. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedeploymentreplicasmismatch + expr: |- + kube_deployment_spec_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_deployment_status_replicas_available{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + for: 1h + labels: + severity: critical + - alert: KubeStatefulSetReplicasMismatch + annotations: + message: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetreplicasmismatch + expr: |- + kube_statefulset_status_replicas_ready{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + for: 15m + labels: + severity: critical + - alert: KubeStatefulSetGenerationMismatch + annotations: + message: StatefulSet generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetgenerationmismatch + expr: |- + kube_statefulset_status_observed_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_metadata_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + for: 15m + labels: + severity: critical + - alert: KubeStatefulSetUpdateNotRolledOut + annotations: + message: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} update has not been rolled out. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetupdatenotrolledout + expr: |- + max without (revision) ( + kube_statefulset_status_current_revision{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + unless + kube_statefulset_status_update_revision{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) + * + ( + kube_statefulset_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + != + kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + ) + for: 15m + labels: + severity: critical + - alert: KubeDaemonSetRolloutStuck + annotations: + message: Only {{`{{`}} $value {{`}}`}}% of the desired Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are scheduled and ready. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetrolloutstuck + expr: |- + kube_daemonset_status_number_ready{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + / + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} * 100 < 100 + for: 15m + labels: + severity: critical + - alert: KubeDaemonSetNotScheduled + annotations: + message: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are not scheduled.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetnotscheduled + expr: |- + kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} + - + kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0 + for: 10m + labels: + severity: warning + - alert: KubeDaemonSetMisScheduled + annotations: + message: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are running where they are not supposed to run.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetmisscheduled + expr: kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0 + for: 10m + labels: + severity: warning + - alert: KubeCronJobRunning + annotations: + message: CronJob {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.cronjob {{`}}`}} is taking more than 1h to complete. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecronjobrunning + expr: time() - kube_cronjob_next_schedule_time{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 3600 + for: 1h + labels: + severity: warning + - alert: KubeJobCompletion + annotations: + message: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} is taking more than one hour to complete. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubejobcompletion + expr: kube_job_spec_completions{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - kube_job_status_succeeded{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0 + for: 1h + labels: + severity: warning + - alert: KubeJobFailed + annotations: + message: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} failed to complete. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubejobfailed + expr: kube_job_status_failed{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0 + for: 1h + labels: + severity: warning +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-resources.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-resources.yaml new file mode 100755 index 000000000..885bc4138 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-resources.yaml @@ -0,0 +1,103 @@ +{{- /* +Generated from 'kubernetes-resources' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesResources }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-resources" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-resources + rules: + - alert: KubeCPUOvercommit + annotations: + message: Cluster has overcommitted CPU resource requests for Pods and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecpuovercommit + expr: |- + sum(namespace_name:kube_pod_container_resource_requests_cpu_cores:sum) + / + sum(node:node_num_cpu:sum) + > + (count(node:node_num_cpu:sum)-1) / count(node:node_num_cpu:sum) + for: 5m + labels: + severity: warning + - alert: KubeMemOvercommit + annotations: + message: Cluster has overcommitted memory resource requests for Pods and cannot tolerate node failure. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubememovercommit + expr: |- + sum(namespace_name:kube_pod_container_resource_requests_memory_bytes:sum) + / + sum(node_memory_MemTotal_bytes) + > + (count(node:node_num_cpu:sum)-1) + / + count(node:node_num_cpu:sum) + for: 5m + labels: + severity: warning + - alert: KubeCPUOvercommit + annotations: + message: Cluster has overcommitted CPU resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecpuovercommit + expr: |- + sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="cpu"}) + / + sum(node:node_num_cpu:sum) + > 1.5 + for: 5m + labels: + severity: warning + - alert: KubeMemOvercommit + annotations: + message: Cluster has overcommitted memory resource requests for Namespaces. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubememovercommit + expr: |- + sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="memory"}) + / + sum(node_memory_MemTotal_bytes{job="node-exporter"}) + > 1.5 + for: 5m + labels: + severity: warning + - alert: KubeQuotaExceeded + annotations: + message: Namespace {{`{{`}} $labels.namespace {{`}}`}} is using {{`{{`}} printf "%0.0f" $value {{`}}`}}% of its {{`{{`}} $labels.resource {{`}}`}} quota. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubequotaexceeded + expr: |- + 100 * kube_resourcequota{job="kube-state-metrics", type="used"} + / ignoring(instance, job, type) + (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0) + > 90 + for: 15m + labels: + severity: warning + - alert: CPUThrottlingHigh + annotations: + message: '{{`{{`}} printf "%0.0f" $value {{`}}`}}% throttling of CPU in namespace {{`{{`}} $labels.namespace {{`}}`}} for container {{`{{`}} $labels.container_name {{`}}`}} in pod {{`{{`}} $labels.pod_name {{`}}`}}.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-cputhrottlinghigh + expr: |- + 100 * sum(increase(container_cpu_cfs_throttled_periods_total{container_name!="", }[5m])) by (container_name, pod_name, namespace) + / + sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container_name, pod_name, namespace) + > 25 + for: 15m + labels: + severity: warning +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-storage.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-storage.yaml new file mode 100755 index 000000000..b2b11e3ec --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-storage.yaml @@ -0,0 +1,63 @@ +{{- /* +Generated from 'kubernetes-storage' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesStorage }} +{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-storage" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-storage + rules: + - alert: KubePersistentVolumeUsageCritical + annotations: + message: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is only {{`{{`}} printf "%0.2f" $value {{`}}`}}% free. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumeusagecritical + expr: |- + 100 * kubelet_volume_stats_available_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}"} + / + kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}"} + < 3 + for: 1m + labels: + severity: critical + - alert: KubePersistentVolumeFullInFourDays + annotations: + message: Based on recent sampling, the PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} is expected to fill up within four days. Currently {{`{{`}} printf "%0.2f" $value {{`}}`}}% is available. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumefullinfourdays + expr: |- + 100 * ( + kubelet_volume_stats_available_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}"} + / + kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}"} + ) < 15 + and + predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", namespace=~"{{ $targetNamespace }}"}[6h], 4 * 24 * 3600) < 0 + for: 5m + labels: + severity: critical + - alert: KubePersistentVolumeErrors + annotations: + message: The persistent volume {{`{{`}} $labels.persistentvolume {{`}}`}} has status {{`{{`}} $labels.phase {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepersistentvolumeerrors + expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0 + for: 5m + labels: + severity: critical +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-system.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-system.yaml new file mode 100755 index 000000000..dac4e7c15 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/kubernetes-system.yaml @@ -0,0 +1,145 @@ +{{- /* +Generated from 'kubernetes-system' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: kubernetes-system + rules: + - alert: KubeNodeNotReady + annotations: + message: '{{`{{`}} $labels.node {{`}}`}} has been unready for more than an hour.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubenodenotready + expr: kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0 + for: 1h + labels: + severity: warning + - alert: KubeVersionMismatch + annotations: + message: There are {{`{{`}} $value {{`}}`}} different semantic versions of Kubernetes components running. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeversionmismatch + expr: count(count by (gitVersion) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"gitVersion","$1","gitVersion","(v[0-9]*.[0-9]*.[0-9]*).*"))) > 1 + for: 1h + labels: + severity: warning + - alert: KubeClientErrors + annotations: + message: Kubernetes API server client '{{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.instance {{`}}`}}' is experiencing {{`{{`}} printf "%0.0f" $value {{`}}`}}% errors.' + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclienterrors + expr: |- + (sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (instance, job) + / + sum(rate(rest_client_requests_total[5m])) by (instance, job)) + * 100 > 1 + for: 15m + labels: + severity: warning + - alert: KubeClientErrors + annotations: + message: Kubernetes API server client '{{`{{`}} $labels.job {{`}}`}}/{{`{{`}} $labels.instance {{`}}`}}' is experiencing {{`{{`}} printf "%0.0f" $value {{`}}`}} errors / second. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclienterrors + expr: sum(rate(ksm_scrape_error_total{job="kube-state-metrics"}[5m])) by (instance, job) > 0.1 + for: 15m + labels: + severity: warning + - alert: KubeletTooManyPods + annotations: + message: Kubelet {{`{{`}} $labels.instance {{`}}`}} is running {{`{{`}} $value {{`}}`}} Pods, close to the limit of 110. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubelettoomanypods + expr: kubelet_running_pod_count{job="kubelet"} > 110 * 0.9 + for: 15m + labels: + severity: warning + - alert: KubeAPILatencyHigh + annotations: + message: The API server has a 99th percentile latency of {{`{{`}} $value {{`}}`}} seconds for {{`{{`}} $labels.verb {{`}}`}} {{`{{`}} $labels.resource {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapilatencyhigh + expr: cluster_quantile:apiserver_request_latencies:histogram_quantile{job="apiserver",quantile="0.99",subresource!="log",verb!~"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$"} > 1 + for: 10m + labels: + severity: warning + - alert: KubeAPILatencyHigh + annotations: + message: The API server has a 99th percentile latency of {{`{{`}} $value {{`}}`}} seconds for {{`{{`}} $labels.verb {{`}}`}} {{`{{`}} $labels.resource {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapilatencyhigh + expr: cluster_quantile:apiserver_request_latencies:histogram_quantile{job="apiserver",quantile="0.99",subresource!="log",verb!~"^(?:LIST|WATCH|WATCHLIST|PROXY|CONNECT)$"} > 4 + for: 10m + labels: + severity: critical + - alert: KubeAPIErrorsHigh + annotations: + message: API server is returning errors for {{`{{`}} $value {{`}}`}}% of requests. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh + expr: |- + sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[5m])) + / + sum(rate(apiserver_request_count{job="apiserver"}[5m])) * 100 > 3 + for: 10m + labels: + severity: critical + - alert: KubeAPIErrorsHigh + annotations: + message: API server is returning errors for {{`{{`}} $value {{`}}`}}% of requests. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh + expr: |- + sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[5m])) + / + sum(rate(apiserver_request_count{job="apiserver"}[5m])) * 100 > 1 + for: 10m + labels: + severity: warning + - alert: KubeAPIErrorsHigh + annotations: + message: API server is returning errors for {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.verb {{`}}`}} {{`{{`}} $labels.resource {{`}}`}} {{`{{`}} $labels.subresource {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh + expr: |- + sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[5m])) by (resource,subresource,verb) + / + sum(rate(apiserver_request_count{job="apiserver"}[5m])) by (resource,subresource,verb) * 100 > 10 + for: 10m + labels: + severity: critical + - alert: KubeAPIErrorsHigh + annotations: + message: API server is returning errors for {{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.verb {{`}}`}} {{`{{`}} $labels.resource {{`}}`}} {{`{{`}} $labels.subresource {{`}}`}}. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeapierrorshigh + expr: |- + sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[5m])) by (resource,subresource,verb) + / + sum(rate(apiserver_request_count{job="apiserver"}[5m])) by (resource,subresource,verb) * 100 > 5 + for: 10m + labels: + severity: warning + - alert: KubeClientCertificateExpiration + annotations: + message: A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclientcertificateexpiration + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 + labels: + severity: warning + - alert: KubeClientCertificateExpiration + annotations: + message: A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours. + runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubeclientcertificateexpiration + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 + labels: + severity: critical +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/node-network.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/node-network.yaml new file mode 100755 index 000000000..d3c853f4a --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/node-network.yaml @@ -0,0 +1,48 @@ +{{- /* +Generated from 'node-network' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.network }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-network" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-network + rules: + - alert: NetworkReceiveErrors + annotations: + message: Network interface "{{`{{`}} $labels.device {{`}}`}}" showing receive errors on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}}" + expr: rate(node_network_receive_errs_total{job="node-exporter",device!~"veth.+"}[2m]) > 0 + for: 2m + labels: + severity: warning + - alert: NetworkTransmitErrors + annotations: + message: Network interface "{{`{{`}} $labels.device {{`}}`}}" showing transmit errors on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}}" + expr: rate(node_network_transmit_errs_total{job="node-exporter",device!~"veth.+"}[2m]) > 0 + for: 2m + labels: + severity: warning + - alert: NodeNetworkInterfaceFlapping + annotations: + message: Network interface "{{`{{`}} $labels.device {{`}}`}}" changing it's up status often on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}}" + expr: changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2 + for: 2m + labels: + severity: warning +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/node-time.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/node-time.yaml new file mode 100755 index 000000000..9c30ea4f1 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/node-time.yaml @@ -0,0 +1,34 @@ +{{- /* +Generated from 'node-time' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.time }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-time" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node-time + rules: + - alert: ClockSkewDetected + annotations: + message: Clock skew detected on node-exporter {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}}. Ensure NTP is configured correctly on this host. + expr: abs(node_timex_offset_seconds{job="node-exporter"}) > 0.03 + for: 2m + labels: + severity: warning +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/node.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/node.rules.yaml new file mode 100755 index 000000000..328c6d7d8 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/node.rules.yaml @@ -0,0 +1,202 @@ +{{- /* +Generated from 'node.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.nodeExporter.enabled .Values.defaultRules.rules.node }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: node.rules + rules: + - expr: sum(min(kube_pod_info) by (node)) + record: ':kube_pod_info_node_count:' + - expr: max(label_replace(kube_pod_info{job="kube-state-metrics"}, "pod", "$1", "pod", "(.*)")) by (node, namespace, pod) + record: 'node_namespace_pod:kube_pod_info:' + - expr: |- + count by (node) (sum by (node, cpu) ( + node_cpu_seconds_total{job="node-exporter"} + * on (namespace, pod) group_left(node) + node_namespace_pod:kube_pod_info: + )) + record: node:node_num_cpu:sum + - expr: 1 - avg(rate(node_cpu_seconds_total{job="node-exporter",mode="idle"}[1m])) + record: :node_cpu_utilisation:avg1m + - expr: |- + 1 - avg by (node) ( + rate(node_cpu_seconds_total{job="node-exporter",mode="idle"}[1m]) + * on (namespace, pod) group_left(node) + node_namespace_pod:kube_pod_info:) + record: node:node_cpu_utilisation:avg1m + - expr: |- + node:node_cpu_utilisation:avg1m + * + node:node_num_cpu:sum + / + scalar(sum(node:node_num_cpu:sum)) + record: node:cluster_cpu_utilisation:ratio + - expr: |- + sum(node_load1{job="node-exporter"}) + / + sum(node:node_num_cpu:sum) + record: ':node_cpu_saturation_load1:' + - expr: |- + sum by (node) ( + node_load1{job="node-exporter"} + * on (namespace, pod) group_left(node) + node_namespace_pod:kube_pod_info: + ) + / + node:node_num_cpu:sum + record: 'node:node_cpu_saturation_load1:' + - expr: |- + 1 - + sum(node_memory_MemFree_bytes{job="node-exporter"} + node_memory_Cached_bytes{job="node-exporter"} + node_memory_Buffers_bytes{job="node-exporter"}) + / + sum(node_memory_MemTotal_bytes{job="node-exporter"}) + record: ':node_memory_utilisation:' + - expr: sum(node_memory_MemFree_bytes{job="node-exporter"} + node_memory_Cached_bytes{job="node-exporter"} + node_memory_Buffers_bytes{job="node-exporter"}) + record: :node_memory_MemFreeCachedBuffers_bytes:sum + - expr: sum(node_memory_MemTotal_bytes{job="node-exporter"}) + record: :node_memory_MemTotal_bytes:sum + - expr: |- + sum by (node) ( + (node_memory_MemFree_bytes{job="node-exporter"} + node_memory_Cached_bytes{job="node-exporter"} + node_memory_Buffers_bytes{job="node-exporter"}) + * on (namespace, pod) group_left(node) + node_namespace_pod:kube_pod_info: + ) + record: node:node_memory_bytes_available:sum + - expr: |- + sum by (node) ( + node_memory_MemTotal_bytes{job="node-exporter"} + * on (namespace, pod) group_left(node) + node_namespace_pod:kube_pod_info: + ) + record: node:node_memory_bytes_total:sum + - expr: |- + (node:node_memory_bytes_total:sum - node:node_memory_bytes_available:sum) + / + node:node_memory_bytes_total:sum + record: node:node_memory_utilisation:ratio + - expr: |- + (node:node_memory_bytes_total:sum - node:node_memory_bytes_available:sum) + / + scalar(sum(node:node_memory_bytes_total:sum)) + record: node:cluster_memory_utilisation:ratio + - expr: |- + 1e3 * sum( + (rate(node_vmstat_pgpgin{job="node-exporter"}[1m]) + + rate(node_vmstat_pgpgout{job="node-exporter"}[1m])) + ) + record: :node_memory_swap_io_bytes:sum_rate + - expr: |- + 1 - + sum by (node) ( + (node_memory_MemFree_bytes{job="node-exporter"} + node_memory_Cached_bytes{job="node-exporter"} + node_memory_Buffers_bytes{job="node-exporter"}) + * on (namespace, pod) group_left(node) + node_namespace_pod:kube_pod_info: + ) + / + sum by (node) ( + node_memory_MemTotal_bytes{job="node-exporter"} + * on (namespace, pod) group_left(node) + node_namespace_pod:kube_pod_info: + ) + record: 'node:node_memory_utilisation:' + - expr: 1 - (node:node_memory_bytes_available:sum / node:node_memory_bytes_total:sum) + record: 'node:node_memory_utilisation_2:' + - expr: |- + 1e3 * sum by (node) ( + (rate(node_vmstat_pgpgin{job="node-exporter"}[1m]) + + rate(node_vmstat_pgpgout{job="node-exporter"}[1m])) + * on (namespace, pod) group_left(node) + node_namespace_pod:kube_pod_info: + ) + record: node:node_memory_swap_io_bytes:sum_rate + - expr: avg(irate(node_disk_io_time_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m])) + record: :node_disk_utilisation:avg_irate + - expr: |- + avg by (node) ( + irate(node_disk_io_time_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m]) + * on (namespace, pod) group_left(node) + node_namespace_pod:kube_pod_info: + ) + record: node:node_disk_utilisation:avg_irate + - expr: avg(irate(node_disk_io_time_weighted_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m])) + record: :node_disk_saturation:avg_irate + - expr: |- + avg by (node) ( + irate(node_disk_io_time_weighted_seconds_total{job="node-exporter",device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m]) + * on (namespace, pod) group_left(node) + node_namespace_pod:kube_pod_info: + ) + record: node:node_disk_saturation:avg_irate + - expr: |- + max by (instance, namespace, pod, device) ((node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"} + - node_filesystem_avail_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"}) + / node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"}) + record: 'node:node_filesystem_usage:' + - expr: max by (instance, namespace, pod, device) (node_filesystem_avail_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"} / node_filesystem_size_bytes{fstype=~"ext[234]|btrfs|xfs|zfs"}) + record: 'node:node_filesystem_avail:' + - expr: |- + sum(irate(node_network_receive_bytes_total{job="node-exporter",device!~"veth.+"}[1m])) + + sum(irate(node_network_transmit_bytes_total{job="node-exporter",device!~"veth.+"}[1m])) + record: :node_net_utilisation:sum_irate + - expr: |- + sum by (node) ( + (irate(node_network_receive_bytes_total{job="node-exporter",device!~"veth.+"}[1m]) + + irate(node_network_transmit_bytes_total{job="node-exporter",device!~"veth.+"}[1m])) + * on (namespace, pod) group_left(node) + node_namespace_pod:kube_pod_info: + ) + record: node:node_net_utilisation:sum_irate + - expr: |- + sum(irate(node_network_receive_drop_total{job="node-exporter",device!~"veth.+"}[1m])) + + sum(irate(node_network_transmit_drop_total{job="node-exporter",device!~"veth.+"}[1m])) + record: :node_net_saturation:sum_irate + - expr: |- + sum by (node) ( + (irate(node_network_receive_drop_total{job="node-exporter",device!~"veth.+"}[1m]) + + irate(node_network_transmit_drop_total{job="node-exporter",device!~"veth.+"}[1m])) + * on (namespace, pod) group_left(node) + node_namespace_pod:kube_pod_info: + ) + record: node:node_net_saturation:sum_irate + - expr: |- + max( + max( + kube_pod_info{job="kube-state-metrics", host_ip!=""} + ) by (node, host_ip) + * on (host_ip) group_right (node) + label_replace( + (max(node_filesystem_files{job="node-exporter", mountpoint="/"}) by (instance)), "host_ip", "$1", "instance", "(.*):.*" + ) + ) by (node) + record: 'node:node_inodes_total:' + - expr: |- + max( + max( + kube_pod_info{job="kube-state-metrics", host_ip!=""} + ) by (node, host_ip) + * on (host_ip) group_right (node) + label_replace( + (max(node_filesystem_files_free{job="node-exporter", mountpoint="/"}) by (instance)), "host_ip", "$1", "instance", "(.*):.*" + ) + ) by (node) + record: 'node:node_inodes_free:' +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/prometheus-operator.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/prometheus-operator.yaml new file mode 100755 index 000000000..091324459 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/prometheus-operator.yaml @@ -0,0 +1,43 @@ +{{- /* +Generated from 'prometheus-operator' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheusOperator }} +{{- $operatorJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "operator" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kube-prometheus" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus-operator + rules: + - alert: PrometheusOperatorReconcileErrors + annotations: + message: Errors while reconciling {{`{{`}} $labels.controller {{`}}`}} in {{`{{`}} $labels.namespace {{`}}`}} Namespace. + expr: rate(prometheus_operator_reconcile_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0.1 + for: 10m + labels: + severity: warning + - alert: PrometheusOperatorNodeLookupErrors + annotations: + message: Errors while reconciling Prometheus in {{`{{`}} $labels.namespace {{`}}`}} Namespace. + expr: rate(prometheus_operator_node_address_lookup_errors_total{job="{{ $operatorJob }}",namespace="{{ $namespace }}"}[5m]) > 0.1 + for: 10m + labels: + severity: warning +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/prometheus.rules.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/prometheus.rules.yaml new file mode 100755 index 000000000..3e4b2bd60 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/rules/prometheus.rules.yaml @@ -0,0 +1,109 @@ +{{- /* +Generated from 'prometheus.rules' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml +Do not change in-place! In order to change this file first read following link: +https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack +*/ -}} +{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} +{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.prometheus }} +{{- $prometheusJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }} +{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus.rules" | trunc 63 | trimSuffix "-" }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.defaultRules.labels }} +{{ toYaml .Values.defaultRules.labels | indent 4 }} +{{- end }} +{{- if .Values.defaultRules.annotations }} + annotations: +{{ toYaml .Values.defaultRules.annotations | indent 4 }} +{{- end }} +spec: + groups: + - name: prometheus.rules + rules: + - alert: PrometheusConfigReloadFailed + annotations: + description: Reloading Prometheus' configuration has failed for {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} + summary: Reloading Prometheus' configuration failed + expr: prometheus_config_last_reload_successful{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"} == 0 + for: 10m + labels: + severity: warning + - alert: PrometheusNotificationQueueRunningFull + annotations: + description: Prometheus' alert notification queue is running full for {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} + summary: Prometheus' alert notification queue is running full + expr: predict_linear(prometheus_notifications_queue_length{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m], 60 * 30) > prometheus_notifications_queue_capacity{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"} + for: 10m + labels: + severity: warning + - alert: PrometheusErrorSendingAlerts + annotations: + description: Errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} to Alertmanager {{`{{`}}$labels.Alertmanager{{`}}`}} + summary: Errors while sending alert from Prometheus + expr: rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) / rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0.01 + for: 10m + labels: + severity: warning + - alert: PrometheusErrorSendingAlerts + annotations: + description: Errors while sending alerts from Prometheus {{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} to Alertmanager {{`{{`}}$labels.Alertmanager{{`}}`}} + summary: Errors while sending alerts from Prometheus + expr: rate(prometheus_notifications_errors_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) / rate(prometheus_notifications_sent_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0.03 + for: 10m + labels: + severity: critical + - alert: PrometheusNotConnectedToAlertmanagers + annotations: + description: Prometheus {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} is not connected to any Alertmanagers + summary: Prometheus is not connected to any Alertmanagers + expr: prometheus_notifications_alertmanagers_discovered{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"} < 1 + for: 10m + labels: + severity: warning + - alert: PrometheusTSDBReloadsFailing + annotations: + description: '{{`{{`}}$labels.job{{`}}`}} at {{`{{`}}$labels.instance{{`}}`}} had {{`{{`}}$value | humanize{{`}}`}} reload failures over the last four hours.' + summary: Prometheus has issues reloading data blocks from disk + expr: increase(prometheus_tsdb_reloads_failures_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[2h]) > 0 + for: 12h + labels: + severity: warning + - alert: PrometheusTSDBCompactionsFailing + annotations: + description: '{{`{{`}}$labels.job{{`}}`}} at {{`{{`}}$labels.instance{{`}}`}} had {{`{{`}}$value | humanize{{`}}`}} compaction failures over the last four hours.' + summary: Prometheus has issues compacting sample blocks + expr: increase(prometheus_tsdb_compactions_failed_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[2h]) > 0 + for: 12h + labels: + severity: warning + - alert: PrometheusTSDBWALCorruptions + annotations: + description: '{{`{{`}}$labels.job{{`}}`}} at {{`{{`}}$labels.instance{{`}}`}} has a corrupted write-ahead log (WAL).' + summary: Prometheus write-ahead log is corrupted + expr: prometheus_tsdb_wal_corruptions_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"} > 0 + for: 4h + labels: + severity: warning + - alert: PrometheusNotIngestingSamples + annotations: + description: Prometheus {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod{{`}}`}} isn't ingesting samples. + summary: Prometheus isn't ingesting samples + expr: rate(prometheus_tsdb_head_samples_appended_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) <= 0 + for: 10m + labels: + severity: warning + - alert: PrometheusTargetScrapesDuplicate + annotations: + description: '{{`{{`}}$labels.namespace{{`}}`}}/{{`{{`}}$labels.pod{{`}}`}} has many samples rejected due to duplicate timestamps but different values' + summary: Prometheus has many samples rejected + expr: increase(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="{{ $prometheusJob }}",namespace="{{ $namespace }}"}[5m]) > 0 + for: 10m + labels: + severity: warning +{{- end }} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/service.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/service.yaml new file mode 100755 index 000000000..cb831c7a2 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/service.yaml @@ -0,0 +1,52 @@ +{{- if .Values.prometheus.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + self-monitor: {{ .Values.prometheus.serviceMonitor.selfMonitor | quote }} +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.service.labels }} +{{ toYaml .Values.prometheus.service.labels | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.annotations }} + annotations: +{{ toYaml .Values.prometheus.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheus.service.clusterIP }} + clusterIP: {{ .Values.prometheus.service.clusterIP }} +{{- end }} +{{- if .Values.prometheus.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheus.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} + ports: + - name: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.prometheus.service.nodePort }} + {{- end }} + port: {{ .Values.prometheus.service.port }} + targetPort: {{ .Values.prometheus.service.targetPort }} +{{- if .Values.prometheus.service.additionalPorts }} +{{ toYaml .Values.prometheus.service.additionalPorts | indent 2 }} +{{- end }} + selector: + app: prometheus + prometheus: {{ template "kube-prometheus-stack.fullname" . }}-prometheus +{{- if .Values.prometheus.service.sessionAffinity }} + sessionAffinity: {{ .Values.prometheus.service.sessionAffinity }} +{{- end }} + type: "{{ .Values.prometheus.service.type }}" +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/serviceaccount.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/serviceaccount.yaml new file mode 100755 index 000000000..862d5f8e4 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/serviceaccount.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kube-prometheus-stack.prometheus.serviceAccountName" . }} + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +{{- if .Values.prometheus.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.prometheus.serviceAccount.annotations | indent 4 }} +{{- end }} +imagePullSecrets: +{{ toYaml .Values.global.imagePullSecrets | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/servicemonitor.yaml new file mode 100755 index 000000000..356c013ff --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/servicemonitor.yaml @@ -0,0 +1,42 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.serviceMonitor.selfMonitor }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-prometheus + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-prometheus + release: {{ $.Release.Name | quote }} + self-monitor: "true" + namespaceSelector: + matchNames: + - {{ printf "%s" (include "kube-prometheus-stack.namespace" .) | quote }} + endpoints: + - port: {{ .Values.prometheus.prometheusSpec.portName }} + {{- if .Values.prometheus.serviceMonitor.interval }} + interval: {{ .Values.prometheus.serviceMonitor.interval }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.scheme }} + scheme: {{ .Values.prometheus.serviceMonitor.scheme }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.tlsConfig }} + tlsConfig: {{ toYaml .Values.prometheus.serviceMonitor.tlsConfig | nindent 6 }} + {{- end }} + {{- if .Values.prometheus.serviceMonitor.bearerTokenFile }} + bearerTokenFile: {{ .Values.prometheus.serviceMonitor.bearerTokenFile }} + {{- end }} + path: "{{ trimSuffix "/" .Values.prometheus.prometheusSpec.routePrefix }}/metrics" +{{- if .Values.prometheus.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.prometheus.serviceMonitor.metricRelabelings | indent 6) . }} +{{- end }} +{{- if .Values.prometheus.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.prometheus.serviceMonitor.relabelings | indent 6 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/servicemonitors.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/servicemonitors.yaml new file mode 100755 index 000000000..959df2124 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/servicemonitors.yaml @@ -0,0 +1,34 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.additionalServiceMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalServiceMonitors }} + - apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: + name: {{ .name }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ template "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + endpoints: +{{ toYaml .endpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + {{- if .namespaceSelector }} + namespaceSelector: +{{ toYaml .namespaceSelector | indent 8 }} + {{- end }} + selector: +{{ toYaml .selector | indent 8 }} + {{- if .targetLabels }} + targetLabels: +{{ toYaml .targetLabels | indent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/serviceperreplica.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/serviceperreplica.yaml new file mode 100755 index 000000000..1a5543362 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/prometheus/serviceperreplica.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.prometheus.enabled .Values.prometheus.servicePerReplica.enabled }} +{{- $count := .Values.prometheus.prometheusSpec.replicas | int -}} +{{- $serviceValues := .Values.prometheus.servicePerReplica -}} +apiVersion: v1 +kind: List +metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-serviceperreplica + namespace: {{ template "kube-prometheus-stack.namespace" . }} +items: +{{- range $i, $e := until $count }} + - apiVersion: v1 + kind: Service + metadata: + name: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + namespace: {{ template "kube-prometheus-stack.namespace" $ }} + labels: + app: {{ include "kube-prometheus-stack.name" $ }}-prometheus +{{ include "kube-prometheus-stack.labels" $ | indent 8 }} + {{- if $serviceValues.annotations }} + annotations: +{{ toYaml $serviceValues.annotations | indent 8 }} + {{- end }} + spec: + {{- if $serviceValues.clusterIP }} + clusterIP: {{ $serviceValues.clusterIP }} + {{- end }} + {{- if $serviceValues.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := $serviceValues.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} + {{- end }} + ports: + - name: {{ $.Values.prometheus.prometheusSpec.portName }} + {{- if eq $serviceValues.type "NodePort" }} + nodePort: {{ $serviceValues.nodePort }} + {{- end }} + port: {{ $serviceValues.port }} + targetPort: {{ $serviceValues.targetPort }} + selector: + app: prometheus + prometheus: {{ include "kube-prometheus-stack.fullname" $ }}-prometheus + statefulset.kubernetes.io/pod-name: prometheus-{{ include "kube-prometheus-stack.fullname" $ }}-prometheus-{{ $i }} + type: "{{ $serviceValues.type }}" +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/clusterrole.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/clusterrole.yaml new file mode 100755 index 000000000..e6bc9d476 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/clusterrole.yaml @@ -0,0 +1,123 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-admin + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-admin: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-edit + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-edit: "true" + {{- end }} +rules: +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + verbs: + - 'get' + - 'list' + - 'watch' +- apiGroups: + - monitoring.coreos.com + resources: + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-view: "true" + {{- end }} +rules: +- apiGroups: + - monitoring.coreos.com + resources: + - alertmanagers + - prometheuses + - prometheuses/finalizers + - alertmanagers/finalizers + - thanosrulers + - thanosrulers/finalizers + - servicemonitors + - podmonitors + - prometheusrules + - podmonitors + verbs: + - 'get' + - 'list' + - 'watch' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: monitoring-ui-view + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - services/proxy + resourceNames: + - "http:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-prometheus:{{ .Values.prometheus.service.port }}" + - "http:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" + - "https:{{ template "kube-prometheus-stack.fullname" . }}-alertmanager:{{ .Values.alertmanager.service.port }}" + - "http:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" + - "https:{{ include "call-nested" (list . "grafana" "grafana.fullname") }}:{{ .Values.grafana.service.port }}" + verbs: + - 'get' +- apiGroups: + - "" + resourceNames: + - {{ template "kube-prometheus-stack.fullname" . }}-prometheus + - {{ template "kube-prometheus-stack.fullname" . }}-alertmanager + - {{ include "call-nested" (list . "grafana" "grafana.fullname") }} + resources: + - endpoints + verbs: + - list +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/config-role.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/config-role.yaml new file mode 100755 index 000000000..f48ffc827 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/config-role.yaml @@ -0,0 +1,48 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-admin + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-edit + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-config-view + namespace: {{ template "kube-prometheus-stack.namespace" . }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/dashboard-role.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/dashboard-role.yaml new file mode 100755 index 000000000..d2f81976a --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/dashboard-role.yaml @@ -0,0 +1,47 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create .Values.grafana.enabled }} +{{- if .Values.grafana.defaultDashboardsEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-admin + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-edit + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: monitoring-dashboard-view + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + labels: {{ include "kube-prometheus-stack.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/default-dashboard.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/default-dashboard.yaml new file mode 100755 index 000000000..faf6d3baf --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/default-dashboard.yaml @@ -0,0 +1,17 @@ +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "home" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/rancher/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml new file mode 100755 index 000000000..d256576ad --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/exporters/ingress-nginx/service.yaml @@ -0,0 +1,24 @@ +{{- if .Values.ingressNginx.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + jobLabel: ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} + namespace: {{ .Values.ingressNginx.namespace }} +spec: + clusterIP: None + ports: + - name: http-metrics + port: {{ .Values.ingressNginx.service.port }} + protocol: TCP + targetPort: {{ .Values.ingressNginx.service.targetPort }} + selector: + {{- if .Values.ingressNginx.service.selector }} +{{ toYaml .Values.ingressNginx.service.selector | indent 4 }} + {{- else }} + app: ingress-nginx + {{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml new file mode 100755 index 000000000..643778772 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/exporters/ingress-nginx/servicemonitor.yaml @@ -0,0 +1,33 @@ +{{- if .Values.ingressNginx.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "kube-prometheus-stack.fullname" . }}-ingress-nginx + namespace: {{ .Values.ingressNginx.namespace }} + labels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx +{{ include "kube-prometheus-stack.labels" . | indent 4 }} +spec: + jobLabel: jobLabel + selector: + matchLabels: + app: {{ template "kube-prometheus-stack.name" . }}-ingress-nginx + release: {{ $.Release.Name | quote }} + namespaceSelector: + matchNames: + - {{ .Values.ingressNginx.namespace }} + endpoints: + - port: http-metrics + {{- if .Values.ingressNginx.serviceMonitor.interval}} + interval: {{ .Values.ingressNginx.serviceMonitor.interval }} + {{- end }} + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token +{{- if .Values.ingressNginx.serviceMonitor.metricRelabelings }} + metricRelabelings: +{{ tpl (toYaml .Values.ingressNginx.serviceMonitor.metricRelabelings | indent 4) . }} +{{- end }} +{{- if .Values.ingressNginx.serviceMonitor.relabelings }} + relabelings: +{{ toYaml .Values.ingressNginx.serviceMonitor.relabelings | indent 4 }} +{{- end }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/hardened.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/hardened.yaml new file mode 100755 index 000000000..f9bf57c7e --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/hardened.yaml @@ -0,0 +1,124 @@ +{{- $namespaces := dict "_0" .Release.Namespace -}} +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled (not .Values.grafana.defaultDashboards.useExistingNamespace) -}} +{{- $_ := set $namespaces "_1" .Values.grafana.defaultDashboards.namespace -}} +{{- end -}} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + template: + metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa + spec: + serviceAccountName: {{ .Chart.Name }}-patch-sa + securityContext: + runAsNonRoot: true + runAsUser: 1000 + restartPolicy: Never + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + containers: + {{- range $_, $ns := $namespaces }} + - name: patch-sa-{{ $ns }} + image: {{ template "system_default_registry" $ }}{{ $.Values.global.kubectl.repository }}:{{ $.Values.global.kubectl.tag }} + imagePullPolicy: {{ $.Values.global.kubectl.pullPolicy }} + command: ["kubectl", "patch", "serviceaccount", "default", "-p", "{\"automountServiceAccountToken\": false}"] + args: ["-n", "{{ $ns }}"] + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +rules: +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: ['get', 'patch'] +- apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ .Chart.Name }}-patch-sa +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-patch-sa + labels: + app: {{ .Chart.Name }}-patch-sa +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-patch-sa +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ .Chart.Name }}-patch-sa + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Chart.Name }}-patch-sa +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: false + volumes: + - 'secret' +{{- range $_, $ns := $namespaces }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-allow-all + namespace: {{ $ns }} +spec: + podSelector: {} + ingress: + - {} + egress: + - {} + policyTypes: + - Ingress + - Egress +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/ingress-nginx-dashboard.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/ingress-nginx-dashboard.yaml new file mode 100755 index 000000000..20c57dd2a --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/rancher-monitoring/ingress-nginx-dashboard.yaml @@ -0,0 +1,18 @@ +# Source: +{{- if and .Values.grafana.enabled .Values.grafana.defaultDashboardsEnabled .Values.ingressNginx.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Values.grafana.defaultDashboards.namespace }} + name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" $) "ingress-nginx" | trunc 63 | trimSuffix "-" }} + annotations: +{{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }} + labels: + {{- if $.Values.grafana.sidecar.dashboards.label }} + {{ $.Values.grafana.sidecar.dashboards.label }}: "1" + {{- end }} + app: {{ template "kube-prometheus-stack.name" $ }}-grafana +{{ include "kube-prometheus-stack.labels" $ | indent 4 }} +data: +{{ (.Files.Glob "files/ingress-nginx/*").AsConfig | indent 2 }} +{{- end }} diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/validate-install-crd.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/validate-install-crd.yaml new file mode 100755 index 000000000..e72053209 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/templates/validate-install-crd.yaml @@ -0,0 +1,19 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +# {{- $found := dict -}} +# {{- set $found "monitoring.coreos.com/v1/Alertmanager" false -}} +# {{- set $found "monitoring.coreos.com/v1/PodMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/Prometheus" false -}} +# {{- set $found "monitoring.coreos.com/v1/PrometheusRule" false -}} +# {{- set $found "monitoring.coreos.com/v1/ServiceMonitor" false -}} +# {{- set $found "monitoring.coreos.com/v1/ThanosRuler" false -}} +# {{- range .Capabilities.APIVersions -}} +# {{- if hasKey $found (toString .) -}} +# {{- set $found (toString .) true -}} +# {{- end -}} +# {{- end -}} +# {{- range $_, $exists := $found -}} +# {{- if (eq $exists false) -}} +# {{- required "Required CRDs are missing. Please install the corresponding CRD chart before installing this chart." "" -}} +# {{- end -}} +# {{- end -}} +#{{- end -}} \ No newline at end of file diff --git a/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/values.yaml b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/values.yaml new file mode 100755 index 000000000..dd15e31d7 --- /dev/null +++ b/charts/rancher-monitoring/rancher-monitoring/9.4.204-rc09/values.yaml @@ -0,0 +1,2635 @@ +# Default values for kube-prometheus-stack. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Rancher Monitoring Configuration + +## Configuration for prometheus-adapter +## ref: https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-adapter +## +prometheus-adapter: + enabled: true + prometheus: + # Change this if you change the namespaceOverride or nameOverride of prometheus-operator + url: http://rancher-monitoring-prometheus.cattle-monitoring-system.svc + port: 9090 + image: + repository: rancher/mirrored-directxman12-k8s-prometheus-adapter-amd64 + tag: v0.7.0 + pullPolicy: IfNotPresent + pullSecrets: {} + psp: + create: true + +## RKE PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-pushprox +## +rkeControllerManager: + enabled: false + metricsPort: 10252 + component: kube-controller-manager + clients: + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeScheduler: + enabled: false + metricsPort: 10251 + component: kube-scheduler + clients: + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/controlplane: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rkeEtcd: + enabled: false + metricsPort: 2379 + component: kube-etcd + clients: + port: 10014 + https: + enabled: true + certDir: /etc/kubernetes/ssl + certFile: kube-etcd-*.pem + keyFile: kube-etcd-*-key.pem + caCertFile: kube-ca.pem + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## Windows Monitoring (note: currently RKE1 only) +## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-windows-exporter +## Runs https://github.com/prometheus-community/windows_exporter as a DaemonSet +## Relies on the existence of a wins server of version v0.1.0+ on every Windows host to allow +## windows_exporter to run as a host process that can publish host metrics to a port on the Pod +windowsExporter: + enabled: false + +## k3s PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-pushprox +## +k3sServer: + enabled: false + metricsPort: 10249 + component: k3s-server + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## KubeADM PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-pushprox +## +kubeAdmControllerManager: + enabled: false + metricsPort: 10257 + component: kube-controller-manager + clients: + port: 10011 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmScheduler: + enabled: false + metricsPort: 10259 + component: kube-scheduler + clients: + port: 10012 + useLocalhost: true + https: + enabled: true + useServiceAccountCredentials: true + insecureSkipVerify: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmProxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +kubeAdmEtcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +## rke2 PushProx Monitoring +## ref: https://github.com/rancher/charts/tree/dev-v2.5-source/packages/rancher-pushprox +## +rke2ControllerManager: + enabled: false + metricsPort: 10252 + component: kube-controller-manager + clients: + port: 10011 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2Scheduler: + enabled: false + metricsPort: 10251 + component: kube-scheduler + clients: + port: 10012 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2Proxy: + enabled: false + metricsPort: 10249 + component: kube-proxy + clients: + port: 10013 + useLocalhost: true + tolerations: + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + +rke2Etcd: + enabled: false + metricsPort: 2381 + component: kube-etcd + clients: + port: 10014 + useLocalhost: true + nodeSelector: + node-role.kubernetes.io/etcd: "true" + tolerations: + - effect: "NoSchedule" + key: node-role.kubernetes.io/master + operator: "Equal" + +## Component scraping nginx-ingress-controller +## +ingressNginx: + enabled: false + + ## The namespace to search for your nginx-ingress-controller + ## + namespace: ingress-nginx + + service: + port: 9913 + targetPort: 10254 + # selector: + # app: ingress-nginx + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +# Prometheus Operator Configuration + +## Provide a name in place of kube-prometheus-stack for `app:` labels +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +nameOverride: "rancher-monitoring" + +## Override the deployment namespace +## NOTE: If you change this value, you must update the prometheus-adapter.prometheus.url +## +namespaceOverride: "cattle-monitoring-system" + +## Provide a k8s version to auto dashboard import script example: kubeTargetVersionOverride: 1.16.6 +## +kubeTargetVersionOverride: "" + +## Provide a name to substitute for the full names of resources +## +fullnameOverride: "" + +## Labels to apply to all resources +## +commonLabels: {} +# scmhash: abc123 +# myLabel: aakkmd + +## Create default rules for monitoring the cluster +## +defaultRules: + create: true + rules: + alertmanager: true + etcd: true + general: true + k8s: true + kubeApiserver: true + kubeApiserverAvailability: true + kubeApiserverError: true + kubeApiserverSlos: true + kubelet: true + kubePrometheusGeneral: true + kubePrometheusNodeAlerting: true + kubePrometheusNodeRecording: true + kubernetesAbsent: true + kubernetesApps: true + kubernetesResources: true + kubernetesStorage: true + kubernetesSystem: true + kubeScheduler: true + kubeStateMetrics: true + network: true + node: true + prometheus: true + prometheusOperator: true + time: true + + ## Runbook url prefix for default rules + runbookUrl: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md# + ## Reduce app namespace alert scope + appNamespacesTarget: ".*" + + ## Labels for default rules + labels: {} + ## Annotations for default rules + annotations: {} + +## Provide custom recording or alerting rules to be deployed into the cluster. +## +additionalPrometheusRules: [] +# - name: my-rule-file +# groups: +# - name: my_group +# rules: +# - record: my_record +# expr: 100 * my_record + +## +global: + cattle: + systemDefaultRegistry: "" + kubectl: + repository: rancher/kubectl + tag: v1.20.2 + pullPolicy: IfNotPresent + rbac: + ## Create RBAC resources for ServiceAccounts and users + ## + create: true + + userRoles: + ## Create default user ClusterRoles to allow users to interact with Prometheus CRs, ConfigMaps, and Secrets + create: true + ## Aggregate default user ClusterRoles into default k8s ClusterRoles + aggregateToDefaultRoles: true + + pspEnabled: true + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + # - name: "image-pull-secret" + +## Configuration for alertmanager +## ref: https://prometheus.io/docs/alerting/alertmanager/ +## +alertmanager: + + ## Deploy alertmanager + ## + enabled: true + + ## Api that prometheus will use to communicate with alertmanager. Possible values are v1, v2 + ## + apiVersion: v2 + + ## Service account for Alertmanager to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + annotations: {} + + ## Configure pod disruption budgets for Alertmanager + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + + ## Alertmanager configuration directives + ## ref: https://prometheus.io/docs/alerting/configuration/#configuration-file + ## https://prometheus.io/webtools/alerting/routing-tree-editor/ + ## + ## Example Slack Config + ## config: + ## route: + ## group_by: ['job'] + ## group_wait: 30s + ## group_interval: 5m + ## repeat_interval: 3h + ## receiver: 'slack-notifications' + ## receivers: + ## - name: 'slack-notifications' + ## slack_configs: + ## - send_resolved: true + ## text: '{{ template "slack.rancher.text" . }}' + ## api_url: + ## templates: + ## - /etc/alertmanager/config/*.tmpl + config: + global: + resolve_timeout: 5m + route: + group_by: ['job'] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + receiver: 'null' + routes: + - match: + alertname: Watchdog + receiver: 'null' + receivers: + - name: 'null' + templates: + - /etc/alertmanager/config/*.tmpl + + ## Pass the Alertmanager configuration directives through Helm's templating + ## engine. If the Alertmanager configuration contains Alertmanager templates, + ## they'll need to be properly escaped so that they are not interpreted by + ## Helm + ## ref: https://helm.sh/docs/developing_charts/#using-the-tpl-function + ## https://prometheus.io/docs/alerting/configuration/#%3Ctmpl_string%3E + ## https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + tplConfig: false + + ## Alertmanager template files to format alerts + ## ref: https://prometheus.io/docs/alerting/notifications/ + ## https://prometheus.io/docs/alerting/notification_examples/ + ## + templateFiles: + rancher_defaults.tmpl: |- + {{- define "slack.rancher.text" -}} + {{ template "rancher.text_multiple" . }} + {{- end -}} + + {{- define "rancher.text_multiple" -}} + *[GROUP - Details]* + One or more alarms in this group have triggered a notification. + + {{- if gt (len .GroupLabels.Values) 0 }} + *Group Labels:* + {{- range .GroupLabels.SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- if .ExternalURL }} + *Link to AlertManager:* {{ .ExternalURL }} + {{- end }} + + {{- range .Alerts }} + {{ template "rancher.text_single" . }} + {{- end }} + {{- end -}} + + {{- define "rancher.text_single" -}} + {{- if .Labels.alertname }} + *[ALERT - {{ .Labels.alertname }}]* + {{- else }} + *[ALERT]* + {{- end }} + {{- if .Labels.severity }} + *Severity:* `{{ .Labels.severity }}` + {{- end }} + {{- if .Labels.cluster }} + *Cluster:* {{ .Labels.cluster }} + {{- end }} + {{- if .Annotations.summary }} + *Summary:* {{ .Annotations.summary }} + {{- end }} + {{- if .Annotations.message }} + *Message:* {{ .Annotations.message }} + {{- end }} + {{- if .Annotations.description }} + *Description:* {{ .Annotations.description }} + {{- end }} + {{- if .Annotations.runbook_url }} + *Runbook URL:* <{{ .Annotations.runbook_url }}|:spiral_note_pad:> + {{- end }} + {{- with .Labels }} + {{- with .Remove (stringSlice "alertname" "severity" "cluster") }} + {{- if gt (len .) 0 }} + *Additional Labels:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Annotations }} + {{- with .Remove (stringSlice "summary" "message" "description" "runbook_url") }} + {{- if gt (len .) 0 }} + *Additional Annotations:* + {{- range .SortedPairs }} + • *{{ .Name }}:* `{{ .Value }}` + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end -}} + + ingress: + enabled: false + + annotations: {} + + labels: {} + + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - alertmanager.domain.com + + ## Paths to use for ingress rules - one path should match the alertmanagerSpec.routePrefix + ## + paths: [] + # - / + + ## TLS configuration for Alertmanager Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: alertmanager-general-tls + # hosts: + # - alertmanager.example.com + + ## Configuration for Alertmanager secret + ## + secret: + + # Should the Alertmanager Config Secret be cleaned up on an uninstall? + # This is set to false by default to prevent the loss of alerting configuration on an uninstall + # Only used Alertmanager is deployed and alertmanager.alertmanagerSpec.useExistingSecret=false + # + cleanupOnUninstall: false + + # The image used to manage the Alertmanager Config Secret's lifecycle + # Only used Alertmanager is deployed and alertmanager.alertmanagerSpec.useExistingSecret=false + # + image: + repository: rancher/rancher-agent + tag: v2.5.7 + pullPolicy: IfNotPresent + + securityContext: + runAsNonRoot: true + runAsUser: 1000 + + annotations: {} + + ## Configuration for creating an Ingress that will map to each Alertmanager replica service + ## alertmanager.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## Secret name containing the TLS certificate for alertmanager per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "alertmanager" + + ## Configuration for Alertmanager service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Alertmanager Service to listen on + ## + port: 9093 + ## To be used with a proxy extraContainer port + ## + targetPort: 9093 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30903 + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + loadBalancerIP: "" + loadBalancerSourceRanges: [] + ## Service type + ## + type: ClusterIP + + ## Configuration for creating a separate Service for each statefulset Alertmanager replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Alertmanager Service per replica to listen on + ## + port: 9093 + + ## To be used with a proxy extraContainer port + targetPort: 9093 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30904 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "loadbalancer" + loadBalancerSourceRanges: [] + ## Service type + ## + type: ClusterIP + + ## If true, create a serviceMonitor for alertmanager + ## + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + selfMonitor: true + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Settings affecting alertmanagerSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#alertmanagerspec + ## + alertmanagerSpec: + ## Standard object’s metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the Alertmanager pods. + ## + podMetadata: {} + + ## Image of Alertmanager + ## + image: + repository: rancher/mirrored-prom-alertmanager + tag: v0.21.0 + sha: "" + + ## If true then the user will be responsible to provide a secret with alertmanager configuration + ## So when true the config part will be ignored (including templateFiles) and the one in the secret will be used + ## + useExistingSecret: false + + ## Secrets is a list of Secrets in the same namespace as the Alertmanager object, which shall be mounted into the + ## Alertmanager Pods. The Secrets are mounted into /etc/alertmanager/secrets/. + ## + secrets: [] + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Alertmanager object, which shall be mounted into the Alertmanager Pods. + ## The ConfigMaps are mounted into /etc/alertmanager/configmaps/. + ## + configMaps: [] + + ## ConfigSecret is the name of a Kubernetes Secret in the same namespace as the Alertmanager object, which contains configuration for + ## this Alertmanager instance. Defaults to 'alertmanager-' The secret is mounted into /etc/alertmanager/config. + ## + # configSecret: + + ## Define Log Format + # Use logfmt (default) or json-formatted logging + logFormat: logfmt + + ## Log level for Alertmanager to be configured with. + ## + logLevel: info + + ## Size is the expected size of the alertmanager cluster. The controller will eventually make the size of the + ## running cluster equal to the expected size. + replicas: 1 + + ## Time duration Alertmanager shall retain data for. Default is '120h', and must match the regular expression + ## [0-9]+(ms|s|m|h) (milliseconds seconds minutes hours). + ## + retention: 120h + + ## Storage is the definition of how storage will be used by the Alertmanager instances. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/user-guides/storage.md + ## + storage: {} + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + + ## The external URL the Alertmanager instances will be available under. This is necessary to generate correct URLs. This is necessary if Alertmanager is not served from root of a DNS name. string false + ## + externalUrl: + + ## The route prefix Alertmanager registers HTTP handlers for. This is useful, if using ExternalURL and a proxy is rewriting HTTP routes of a request, and the actual ExternalURL is still true, + ## but the server serves requests under a different route prefix. For example for use with kubectl proxy. + ## + routePrefix: / + + ## If set to true all actions on the underlying managed objects are not going to be performed, except for delete actions. + ## + paused: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Define resources requests and limits for single Pods. + ## ref: https://kubernetes.io/docs/user-guide/compute-resources/ + ## + resources: + limits: + memory: 500Mi + cpu: 1000m + requests: + memory: 100Mi + cpu: 100m + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + ## + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the alertmanager instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## If specified, the pod's tolerations. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. *v1.PodSecurityContext false + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + + ## ListenLocal makes the Alertmanager server listen on loopback, so that it does not bind against the Pod IP. + ## Note this is only for the Alertmanager UI, not the gossip communication. + ## + listenLocal: false + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to an Alertmanager pod. + ## + containers: [] + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## AdditionalPeers allows injecting a set of additional Alertmanagers to peer with to form a highly available cluster. + ## + additionalPeers: [] + + ## PortName to use for Alert Manager. + ## + portName: "web" + + +## Using default values from https://github.com/grafana/helm-charts/blob/main/charts/grafana/values.yaml +## +grafana: + enabled: true + namespaceOverride: "" + + ## Grafana's primary configuration + ## NOTE: values in map will be converted to ini format + ## ref: http://docs.grafana.org/installation/configuration/ + ## + grafana.ini: + users: + auto_assign_org_role: Viewer + auth: + disable_login_form: false + auth.anonymous: + enabled: true + org_role: Viewer + auth.basic: + enabled: false + dashboards: + # Modify this value to change the default dashboard shown on the main Grafana page + default_home_dashboard_path: /tmp/dashboards/rancher-default-home.json + + deploymentStrategy: + type: Recreate + + ## Deploy default dashboards. + ## + defaultDashboardsEnabled: true + + # Additional options for defaultDashboards + defaultDashboards: + # The default namespace to place defaultDashboards within + namespace: cattle-dashboards + # Whether to create the default namespace as a Helm managed namespace or use an existing namespace + # If false, the defaultDashboards.namespace will be created as a Helm managed namespace + useExistingNamespace: false + # Whether the Helm managed namespace created by this chart should be left behind on a Helm uninstall + # If you place other dashboards in this namespace, then they will be deleted on a helm uninstall + # Ignore if useExistingNamespace is true + cleanupOnUninstall: false + + adminPassword: prom-operator + + ingress: + ## If true, Grafana Ingress will be created + ## + enabled: false + + ## Annotations for Grafana Ingress + ## + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + + ## Labels to be added to the Ingress + ## + labels: {} + + ## Hostnames. + ## Must be provided if Ingress is enable. + ## + # hosts: + # - grafana.domain.com + hosts: [] + + ## Path for grafana ingress + path: / + + ## TLS configuration for grafana Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: grafana-general-tls + # hosts: + # - grafana.example.com + + sidecar: + dashboards: + enabled: true + label: grafana_dashboard + searchNamespace: cattle-dashboards + + ## Annotations for Grafana dashboard configmaps + ## + annotations: {} + datasources: + enabled: true + defaultDatasourceEnabled: true + + ## Annotations for Grafana datasource configmaps + ## + annotations: {} + + ## Create datasource for each Pod of Prometheus StatefulSet; + ## this uses headless service `prometheus-operated` which is + ## created by Prometheus Operator + ## ref: https://git.io/fjaBS + createPrometheusReplicasDatasources: false + label: grafana_datasource + + extraConfigmapMounts: [] + # - name: certs-configmap + # mountPath: /etc/grafana/ssl/ + # configMap: certs-configmap + # readOnly: true + + ## Configure additional grafana datasources (passed through tpl) + ## ref: http://docs.grafana.org/administration/provisioning/#datasources + additionalDataSources: [] + # - name: prometheus-sample + # access: proxy + # basicAuth: true + # basicAuthPassword: pass + # basicAuthUser: daco + # editable: false + # jsonData: + # tlsSkipVerify: true + # orgId: 1 + # type: prometheus + # url: https://{{ printf "%s-prometheus.svc" .Release.Name }}:9090 + # version: 1 + + ## Passed to grafana subchart and used by servicemonitor below + ## + service: + portName: nginx-http + ## Port for Grafana Service to listen on + ## + port: 80 + ## To be used with a proxy extraContainer port + ## + targetPort: 8080 + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30950 + ## Service type + ## + type: ClusterIP + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.19.2-alpine + + ## Enable an Specify container in extraContainers. This is meant to allow adding an authentication proxy to a grafana pod + extraContainers: | + - name: grafana-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }}" + ports: + - containerPort: 8080 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: grafana-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + ## Volumes that can be used in containers + extraContainerVolumes: + - name: nginx-home + emptyDir: {} + - name: grafana-nginx + configMap: + name: grafana-nginx-proxy-config + items: + - key: nginx.conf + mode: 438 + path: nginx.conf + + ## If true, create a serviceMonitor for grafana + ## + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + selfMonitor: true + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + resources: + limits: + memory: 200Mi + cpu: 200m + requests: + memory: 100Mi + cpu: 100m + +## Component scraping the kube api server +## +kubeApiServer: + enabled: true + tlsConfig: + serverName: kubernetes + insecureSkipVerify: false + + ## If your API endpoint address is not reachable (as in AKS) you can replace it with the kubernetes service + ## + relabelings: [] + # - sourceLabels: + # - __meta_kubernetes_namespace + # - __meta_kubernetes_service_name + # - __meta_kubernetes_endpoint_port_name + # action: keep + # regex: default;kubernetes;https + # - targetLabel: __address__ + # replacement: kubernetes.default.svc:443 + + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + jobLabel: component + selector: + matchLabels: + component: apiserver + provider: kubernetes + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + +## Component scraping the kubelet and kubelet-hosted cAdvisor +## +kubelet: + enabled: true + namespace: kube-system + + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Enable scraping the kubelet over https. For requirements to enable this see + ## https://github.com/prometheus-operator/prometheus-operator/issues/926 + ## + https: true + + ## Enable scraping /metrics/cadvisor from kubelet's service + ## + cAdvisor: true + + ## Enable scraping /metrics/probes from kubelet's service + ## + probes: true + + ## Enable scraping /metrics/resource from kubelet's service + ## + resource: true + # From kubernetes 1.18, /metrics/resource/v1alpha1 renamed to /metrics/resource + resourcePath: "/metrics/resource/v1alpha1" + ## Metric relabellings to apply to samples before ingestion + ## + cAdvisorMetricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + ## Metric relabellings to apply to samples before ingestion + ## + probesMetricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + # relabel configs to apply to samples before ingestion. + # metrics_path is required to match upstream rules and charts + ## + cAdvisorRelabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + probesRelabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + resourceRelabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + metricRelabelings: [] + # - sourceLabels: [__name__, image] + # separator: ; + # regex: container_([a-z_]+); + # replacement: $1 + # action: drop + # - sourceLabels: [__name__] + # separator: ; + # regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) + # replacement: $1 + # action: drop + + # relabel configs to apply to samples before ingestion. + # metrics_path is required to match upstream rules and charts + ## + relabelings: + - sourceLabels: [__metrics_path__] + targetLabel: metrics_path + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +## Component scraping the kube controller manager +## +kubeControllerManager: + enabled: false + + ## If your kube controller manager is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeControllerManager.endpoints only the port and targetPort are used + ## + service: + port: 10252 + targetPort: 10252 + # selector: + # component: kube-controller-manager + + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Enable scraping kube-controller-manager over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks + ## + https: false + + # Skip TLS certificate validation when scraping + insecureSkipVerify: null + + # Name of the server to use when validating TLS certificate + serverName: null + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +## Component scraping coreDns. Use either this or kubeDns +## +coreDns: + enabled: true + service: + port: 9153 + targetPort: 9153 + # selector: + # k8s-app: kube-dns + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +## Component scraping kubeDns. Use either this or coreDns +## +kubeDns: + enabled: false + service: + dnsmasq: + port: 10054 + targetPort: 10054 + skydns: + port: 10055 + targetPort: 10055 + # selector: + # k8s-app: kube-dns + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + dnsmasqMetricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + dnsmasqRelabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +## Component scraping etcd +## +kubeEtcd: + enabled: false + + ## If your etcd is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## Etcd service. If using kubeEtcd.endpoints only the port and targetPort are used + ## + service: + port: 2379 + targetPort: 2379 + # selector: + # component: etcd + + ## Configure secure access to the etcd cluster by loading a secret into prometheus and + ## specifying security configuration below. For example, with a secret named etcd-client-cert + ## + ## serviceMonitor: + ## scheme: https + ## insecureSkipVerify: false + ## serverName: localhost + ## caFile: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + ## certFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client + ## keyFile: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + ## + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + scheme: http + insecureSkipVerify: false + serverName: "" + caFile: "" + certFile: "" + keyFile: "" + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + +## Component scraping kube scheduler +## +kubeScheduler: + enabled: false + + ## If your kube scheduler is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + ## If using kubeScheduler.endpoints only the port and targetPort are used + ## + service: + port: 10251 + targetPort: 10251 + # selector: + # component: kube-scheduler + + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + ## Enable scraping kube-scheduler over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks + ## + https: false + + ## Skip TLS certificate validation when scraping + insecureSkipVerify: null + + ## Name of the server to use when validating TLS certificate + serverName: null + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + +## Component scraping kube proxy +## +kubeProxy: + enabled: false + + ## If your kube proxy is not deployed as a pod, specify IPs it can be found on + ## + endpoints: [] + # - 10.141.4.22 + # - 10.141.4.23 + # - 10.141.4.24 + + service: + port: 10249 + targetPort: 10249 + # selector: + # k8s-app: kube-proxy + + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## Enable scraping kube-proxy over https. + ## Requires proper certs (not self-signed) and delegated authentication/authorization checks + ## + https: false + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + +## Component scraping kube state metrics +## +kubeStateMetrics: + enabled: true + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +## Configuration for kube-state-metrics subchart +## +kube-state-metrics: + namespaceOverride: "" + rbac: + create: true + podSecurityPolicy: + enabled: true + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 100m + memory: 130Mi + +## Deploy node exporter as a daemonset to all nodes +## +nodeExporter: + enabled: true + + ## Use the value configured in prometheus-node-exporter.podLabels + ## + jobLabel: jobLabel + + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + + ## How long until a scrape request times out. If not set, the Prometheus default scape timeout is used. + ## + scrapeTimeout: "" + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - sourceLabels: [__name__] + # separator: ; + # regex: ^node_mountstats_nfs_(event|operations|transport)_.+ + # replacement: $1 + # action: drop + + ## relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + +## Configuration for prometheus-node-exporter subchart +## +prometheus-node-exporter: + namespaceOverride: "" + podLabels: + ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards + ## + jobLabel: node-exporter + extraArgs: + - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/) + - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$ + service: + port: 9796 + targetPort: 9796 + resources: + limits: + cpu: 200m + memory: 50Mi + requests: + cpu: 100m + memory: 30Mi + +## Manages Prometheus and Alertmanager components +## +prometheusOperator: + enabled: true + + # If true prometheus operator will create and update its CRDs on startup + # Only for prometheusOperator.image.tag < v0.39.0 + manageCrds: true + + tlsProxy: + enabled: true + image: + repository: rancher/mirrored-squareup-ghostunnel + tag: v1.5.2 + sha: "" + pullPolicy: IfNotPresent + resources: {} + + ## Admission webhook support for PrometheusRules resources added in Prometheus Operator 0.30 can be enabled to prevent incorrectly formatted + ## rules from making their way into prometheus and potentially preventing the container from starting + admissionWebhooks: + failurePolicy: Fail + enabled: true + ## If enabled, generate a self-signed certificate, then patch the webhook configurations with the generated data. + ## On chart upgrades (or if the secret exists) the cert will not be re-generated. You can use this to provide your own + ## certs ahead of time if you wish. + ## + patch: + enabled: true + image: + repository: rancher/mirrored-jettech-kube-webhook-certgen + tag: v1.2.1 + sha: "" + pullPolicy: IfNotPresent + resources: {} + ## Provide a priority class name to the webhook patching job + ## + priorityClassName: "" + podAnnotations: {} + nodeSelector: {} + affinity: {} + tolerations: [] + + ## Namespaces to scope the interaction of the Prometheus Operator and the apiserver (allow list). + ## This is mutually exclusive with denyNamespaces. Setting this to an empty object will disable the configuration + ## + namespaces: {} + # releaseNamespace: true + # additional: + # - kube-system + + ## Namespaces not to scope the interaction of the Prometheus Operator (deny list). + ## + denyNamespaces: [] + + ## Service account for Alertmanager to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + + ## Configuration for Prometheus operator service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30080 + + nodePortTls: 30443 + + ## Additional ports to open for Prometheus service + ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#multi-port-services + ## + additionalPorts: [] + + ## Loadbalancer IP + ## Only use if service.type is "loadbalancer" + ## + loadBalancerIP: "" + loadBalancerSourceRanges: [] + + ## Service type + ## NodePort, ClusterIP, loadbalancer + ## + type: ClusterIP + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + ## Deploy CRDs used by Prometheus Operator. + ## + createCustomResource: true + + ## Attempt to clean up CRDs created by Prometheus Operator. + ## + cleanupCustomResource: false + + ## Labels to add to the operator pod + ## + podLabels: {} + + ## Annotations to add to the operator pod + ## + podAnnotations: {} + + ## Assign a PriorityClassName to pods if set + # priorityClassName: "" + + ## Define Log Format + # Use logfmt (default) or json-formatted logging + # logFormat: logfmt + + ## Decrease log verbosity to errors only + # logLevel: error + + ## If true, the operator will create and maintain a service for scraping kubelets + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/helm/prometheus-operator/README.md + ## + kubeletService: + enabled: true + namespace: kube-system + + ## Create a servicemonitor for the operator + ## + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + ## Scrape timeout. If not set, the Prometheus default scrape timeout is used. + scrapeTimeout: "" + selfMonitor: true + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Resource limits & requests + ## + resources: + limits: + cpu: 200m + memory: 500Mi + requests: + cpu: 100m + memory: 100Mi + + # Required for use in managed kubernetes clusters (such as AWS EKS) with custom CNI (such as calico), + # because control-plane managed by AWS cannot communicate with pods' IP CIDR and admission webhooks are not working + ## + hostNetwork: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Assign custom affinity rules to the prometheus operator + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + + ## Prometheus-operator image + ## + image: + repository: rancher/mirrored-coreos-prometheus-operator + tag: v0.38.1 + sha: "" + pullPolicy: IfNotPresent + + ## Configmap-reload image to use for reloading configmaps + ## + configmapReloadImage: + repository: rancher/mirrored-jimmidyson-configmap-reload + tag: v0.3.0 + sha: "" + + ## Prometheus-config-reloader image to use for config and rule reloading + ## + prometheusConfigReloaderImage: + repository: rancher/mirrored-coreos-prometheus-config-reloader + tag: v0.38.1 + sha: "" + + ## Set the prometheus config reloader side-car CPU limit + ## + configReloaderCpu: 100m + + ## Set the prometheus config reloader side-car memory limit + ## + configReloaderMemory: 25Mi + + ## Set a Field Selector to filter watched secrets + ## + secretFieldSelector: "" + +## Deploy a Prometheus instance +## +prometheus: + + enabled: true + + ## Annotations for Prometheus + ## + annotations: {} + + ## Service account for Prometheuses to use. + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ## + serviceAccount: + create: true + name: "" + + ## Configuration for Prometheus service + ## + service: + annotations: {} + labels: {} + clusterIP: "" + + ## Port for Prometheus Service to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 8080 + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 30090 + + ## Loadbalancer IP + ## Only use if service.type is "loadbalancer" + loadBalancerIP: "" + loadBalancerSourceRanges: [] + ## Service type + ## + type: ClusterIP + + sessionAffinity: "" + + ## Configuration for creating a separate Service for each statefulset Prometheus replica + ## + servicePerReplica: + enabled: false + annotations: {} + + ## Port for Prometheus Service per replica to listen on + ## + port: 9090 + + ## To be used with a proxy extraContainer port + targetPort: 9090 + + ## Port to expose on each node + ## Only used if servicePerReplica.type is 'NodePort' + ## + nodePort: 30091 + + ## Loadbalancer source IP ranges + ## Only used if servicePerReplica.type is "loadbalancer" + loadBalancerSourceRanges: [] + ## Service type + ## + type: ClusterIP + + ## Configure pod disruption budgets for Prometheus + ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget + ## This configuration is immutable once created and will require the PDB to be deleted to be changed + ## https://github.com/kubernetes/kubernetes/issues/45398 + ## + podDisruptionBudget: + enabled: false + minAvailable: 1 + maxUnavailable: "" + +# Ingress exposes thanos sidecar outside the clsuter + thanosIngress: + enabled: false + annotations: {} + labels: {} + servicePort: 10901 + ## Hosts must be provided if Ingress is enabled. + ## + hosts: [] + # - thanos-gateway.domain.com + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## TLS configuration for Alertmanager Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: thanos-gateway-tls + # hosts: + # - thanos-gateway.domain.com + + ingress: + enabled: false + annotations: {} + labels: {} + + ## Hostnames. + ## Must be provided if Ingress is enabled. + ## + # hosts: + # - prometheus.domain.com + hosts: [] + + ## Paths to use for ingress rules - one path should match the prometheusSpec.routePrefix + ## + paths: [] + # - / + + ## TLS configuration for Prometheus Ingress + ## Secret must be manually created in the namespace + ## + tls: [] + # - secretName: prometheus-general-tls + # hosts: + # - prometheus.example.com + + ## Configuration for creating an Ingress that will map to each Prometheus replica service + ## prometheus.servicePerReplica must be enabled + ## + ingressPerReplica: + enabled: false + annotations: {} + labels: {} + + ## Final form of the hostname for each per replica ingress is + ## {{ ingressPerReplica.hostPrefix }}-{{ $replicaNumber }}.{{ ingressPerReplica.hostDomain }} + ## + ## Prefix for the per replica ingress that will have `-$replicaNumber` + ## appended to the end + hostPrefix: "" + ## Domain that will be used for the per replica ingress + hostDomain: "" + + ## Paths to use for ingress rules + ## + paths: [] + # - / + + ## Secret name containing the TLS certificate for Prometheus per replica ingress + ## Secret must be manually created in the namespace + tlsSecretName: "" + + ## Separated secret for each per replica Ingress. Can be used together with cert-manager + ## + tlsSecretPerReplica: + enabled: false + ## Final form of the secret for each per replica ingress is + ## {{ tlsSecretPerReplica.prefix }}-{{ $replicaNumber }} + ## + prefix: "prometheus" + + ## Configure additional options for default pod security policy for Prometheus + ## ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + allowedCapabilities: [] + + serviceMonitor: + ## Scrape interval. If not set, the Prometheus default scrape interval is used. + ## + interval: "" + selfMonitor: true + + ## scheme: HTTP scheme to use for scraping. Can be used with `tlsConfig` for example if using istio mTLS. + scheme: "" + + ## tlsConfig: TLS configuration to use when scraping the endpoint. For example if using istio mTLS. + ## Of type: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#tlsconfig + tlsConfig: {} + + bearerTokenFile: + + ## metric relabel configs to apply to samples before ingestion. + ## + metricRelabelings: [] + # - action: keep + # regex: 'kube_(daemonset|deployment|pod|namespace|node|statefulset).+' + # sourceLabels: [__name__] + + # relabel configs to apply to samples before ingestion. + ## + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # separator: ; + # regex: ^(.*)$ + # targetLabel: nodename + # replacement: $1 + # action: replace + + ## Settings affecting prometheusSpec + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#prometheusspec + ## + prometheusSpec: + ## If true, pass --storage.tsdb.max-block-duration=2h to prometheus. This is already done if using Thanos + ## + disableCompaction: false + ## APIServerConfig + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#apiserverconfig + ## + apiserverConfig: {} + + ## Interval between consecutive scrapes. + ## + scrapeInterval: "" + + ## Interval between consecutive evaluations. + ## + evaluationInterval: "" + + ## ListenLocal makes the Prometheus server listen on loopback, so that it does not bind against the Pod IP. + ## + listenLocal: false + + ## EnableAdminAPI enables Prometheus the administrative HTTP API which includes functionality such as deleting time series. + ## This is disabled by default. + ## ref: https://prometheus.io/docs/prometheus/latest/querying/api/#tsdb-admin-apis + ## + enableAdminAPI: false + + ## Image of Prometheus. + ## + image: + repository: rancher/mirrored-prom-prometheus + tag: v2.18.2 + sha: "" + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + ## Alertmanagers to which alerts will be sent + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#alertmanagerendpoints + ## + ## Default configuration will connect to the alertmanager deployed as part of this release + ## + alertingEndpoints: [] + # - name: "" + # namespace: "" + # port: http + # scheme: http + # pathPrefix: "" + # tlsConfig: {} + # bearerTokenFile: "" + # apiVersion: v2 + + ## External labels to add to any time series or alerts when communicating with external systems + ## + externalLabels: {} + + ## Name of the external label used to denote replica name + ## + replicaExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote replica name + ## + replicaExternalLabelNameClear: false + + ## Name of the external label used to denote Prometheus instance name + ## + prometheusExternalLabelName: "" + + ## If true, the Operator won't add the external label used to denote Prometheus instance name + ## + prometheusExternalLabelNameClear: false + + ## External URL at which Prometheus will be reachable. + ## + externalUrl: "" + + ## Ignore NamespaceSelector settings from the PodMonitor and ServiceMonitor configs + ## If true, PodMonitors and ServiceMonitors can only discover Pods and Services within the namespace they are deployed into + ## + ignoreNamespaceSelectors: false + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Secrets is a list of Secrets in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The Secrets are mounted into /etc/prometheus/secrets/. Secrets changes after initial creation of a Prometheus object are not + ## reflected in the running Pods. To change the secrets mounted into the Prometheus Pods, the object must be deleted and recreated + ## with the new list of secrets. + ## + secrets: [] + + ## ConfigMaps is a list of ConfigMaps in the same namespace as the Prometheus object, which shall be mounted into the Prometheus Pods. + ## The ConfigMaps are mounted into /etc/prometheus/configmaps/. + ## + configMaps: [] + + ## QuerySpec defines the query command line flags when starting Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#queryspec + ## + query: {} + + ## Namespaces to be selected for PrometheusRules discovery. + ## If nil, select own namespace. Namespaces to be selected for ServiceMonitor discovery. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#namespaceselector for usage + ## + ruleNamespaceSelector: {} + + ## If true, a nil or {} value for prometheus.prometheusSpec.ruleSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the PrometheusRule resources created + ## + ruleSelectorNilUsesHelmValues: false + + ## PrometheusRules to be selected for target discovery. + ## If {}, select all ServiceMonitors + ## + ruleSelector: {} + ## Example which select all prometheusrules resources + ## with label "prometheus" with values any of "example-rules" or "example-rules-2" + # ruleSelector: + # matchExpressions: + # - key: prometheus + # operator: In + # values: + # - example-rules + # - example-rules-2 + # + ## Example which select all prometheusrules resources with label "role" set to "example-rules" + # ruleSelector: + # matchLabels: + # role: example-rules + + ## If true, a nil or {} value for prometheus.prometheusSpec.serviceMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the servicemonitors created + ## + serviceMonitorSelectorNilUsesHelmValues: false + + ## ServiceMonitors to be selected for target discovery. + ## If {}, select all ServiceMonitors + ## + serviceMonitorSelector: {} + ## Example which selects ServiceMonitors with label "prometheus" set to "somelabel" + # serviceMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## Namespaces to be selected for ServiceMonitor discovery. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#namespaceselector for usage + ## + serviceMonitorNamespaceSelector: {} + + ## If true, a nil or {} value for prometheus.prometheusSpec.podMonitorSelector will cause the + ## prometheus resource to be created with selectors based on values in the helm deployment, + ## which will also match the podmonitors created + ## + podMonitorSelectorNilUsesHelmValues: false + + ## PodMonitors to be selected for target discovery. + ## If {}, select all PodMonitors + ## + podMonitorSelector: {} + ## Example which selects PodMonitors with label "prometheus" set to "somelabel" + # podMonitorSelector: + # matchLabels: + # prometheus: somelabel + + ## Namespaces to be selected for PodMonitor discovery. + ## See https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#namespaceselector for usage + ## + podMonitorNamespaceSelector: {} + + ## How long to retain metrics + ## + retention: 10d + + ## Maximum size of metrics + ## + retentionSize: "" + + ## Enable compression of the write-ahead log using Snappy. + ## + walCompression: false + + ## If true, the Operator won't process any Prometheus configuration changes + ## + paused: false + + ## Number of Prometheus replicas desired + ## + replicas: 1 + + ## Log level for Prometheus be configured in + ## + logLevel: info + + ## Log format for Prometheus be configured in + ## + logFormat: logfmt + + ## Prefix used to register routes, overriding externalUrl route. + ## Useful for proxies that rewrite URLs. + ## + routePrefix: / + + ## Standard object’s metadata. More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata + ## Metadata Labels and Annotations gets propagated to the prometheus pods. + ## + podMetadata: {} + # labels: + # app: prometheus + # k8s-app: prometheus + + ## Pod anti-affinity can prevent the scheduler from placing Prometheus replicas on the same node. + ## The default value "soft" means that the scheduler should *prefer* to not schedule two replica pods onto the same node but no guarantee is provided. + ## The value "hard" means that the scheduler is *required* to not schedule two replica pods onto the same node. + ## The value "" will disable pod anti-affinity so that no anti-affinity rules will be configured. + podAntiAffinity: "" + + ## If anti-affinity is enabled sets the topologyKey to use for anti-affinity. + ## This can be changed to, for example, failure-domain.beta.kubernetes.io/zone + ## + podAntiAffinityTopologyKey: kubernetes.io/hostname + + ## Assign custom affinity rules to the prometheus instance + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ + ## + affinity: {} + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/e2e-az-name + # operator: In + # values: + # - e2e-az1 + # - e2e-az2 + + ## The remote_read spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#remotereadspec + remoteRead: [] + # - url: http://remote1/read + + ## The remote_write spec configuration for Prometheus. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#remotewritespec + remoteWrite: [] + # - url: http://remote1/push + + ## Enable/Disable Grafana dashboards provisioning for prometheus remote write feature + remoteWriteDashboards: false + + ## Resource limits & requests + ## + resources: + limits: + memory: 1500Mi + cpu: 1000m + requests: + memory: 750Mi + cpu: 750m + + ## Prometheus StorageSpec for persistent data + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/user-guides/storage.md + ## + storageSpec: {} + # volumeClaimTemplate: + # spec: + # storageClassName: gluster + # accessModes: ["ReadWriteOnce"] + # resources: + # requests: + # storage: 50Gi + # selector: {} + + ## AdditionalScrapeConfigs allows specifying additional Prometheus scrape configurations. Scrape configurations + ## are appended to the configurations generated by the Prometheus Operator. Job configurations must have the form + ## as specified in the official Prometheus documentation: + ## https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. As scrape configs are + ## appended, the user is responsible to make sure it is valid. Note that using this feature may expose the possibility + ## to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible + ## scrape configs are going to break Prometheus after the upgrade. + ## + ## The scrape configuraiton example below will find master nodes, provided they have the name .*mst.*, relabel the + ## port to 2379 and allow etcd scraping provided it is running on all Kubernetes master nodes + ## + additionalScrapeConfigs: [] + # - job_name: kube-etcd + # kubernetes_sd_configs: + # - role: node + # scheme: https + # tls_config: + # ca_file: /etc/prometheus/secrets/etcd-client-cert/etcd-ca + # cert_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client + # key_file: /etc/prometheus/secrets/etcd-client-cert/etcd-client-key + # relabel_configs: + # - action: labelmap + # regex: __meta_kubernetes_node_label_(.+) + # - source_labels: [__address__] + # action: replace + # targetLabel: __address__ + # regex: ([^:;]+):(\d+) + # replacement: ${1}:2379 + # - source_labels: [__meta_kubernetes_node_name] + # action: keep + # regex: .*mst.* + # - source_labels: [__meta_kubernetes_node_name] + # action: replace + # targetLabel: node + # regex: (.*) + # replacement: ${1} + # metric_relabel_configs: + # - regex: (kubernetes_io_hostname|failure_domain_beta_kubernetes_io_region|beta_kubernetes_io_os|beta_kubernetes_io_arch|beta_kubernetes_io_instance_type|failure_domain_beta_kubernetes_io_zone) + # action: labeldrop + + ## If additional scrape configurations are already deployed in a single secret file you can use this section. + ## Expected values are the secret name and key + ## Cannot be used with additionalScrapeConfigs + additionalScrapeConfigsSecret: {} + # enabled: false + # name: + # key: + + ## additionalPrometheusSecretsAnnotations allows to add annotations to the kubernetes secret. This can be useful + ## when deploying via spinnaker to disable versioning on the secret, strategy.spinnaker.io/versioned: 'false' + additionalPrometheusSecretsAnnotations: {} + + ## AdditionalAlertManagerConfigs allows for manual configuration of alertmanager jobs in the form as specified + ## in the official Prometheus documentation https://prometheus.io/docs/prometheus/latest/configuration/configuration/#. + ## AlertManager configurations specified are appended to the configurations generated by the Prometheus Operator. + ## As AlertManager configs are appended, the user is responsible to make sure it is valid. Note that using this + ## feature may expose the possibility to break upgrades of Prometheus. It is advised to review Prometheus release + ## notes to ensure that no incompatible AlertManager configs are going to break Prometheus after the upgrade. + ## + additionalAlertManagerConfigs: [] + # - consul_sd_configs: + # - server: consul.dev.test:8500 + # scheme: http + # datacenter: dev + # tag_separator: ',' + # services: + # - metrics-prometheus-alertmanager + + ## AdditionalAlertRelabelConfigs allows specifying Prometheus alert relabel configurations. Alert relabel configurations specified are appended + ## to the configurations generated by the Prometheus Operator. Alert relabel configurations specified must have the form as specified in the + ## official Prometheus documentation: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alert_relabel_configs. + ## As alert relabel configs are appended, the user is responsible to make sure it is valid. Note that using this feature may expose the + ## possibility to break upgrades of Prometheus. It is advised to review Prometheus release notes to ensure that no incompatible alert relabel + ## configs are going to break Prometheus after the upgrade. + ## + additionalAlertRelabelConfigs: [] + # - separator: ; + # regex: prometheus_replica + # replacement: $1 + # action: labeldrop + + ## SecurityContext holds pod-level security attributes and common container settings. + ## This defaults to non root user with uid 1000 and gid 2000. + ## https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md + ## + securityContext: + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + + ## Priority class assigned to the Pods + ## + priorityClassName: "" + + ## Thanos configuration allows configuring various aspects of a Prometheus server in a Thanos environment. + ## This section is experimental, it may change significantly without deprecation notice in any release. + ## This is experimental and may change significantly without backward compatibility in any release. + ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#thanosspec + ## + thanos: {} + + proxy: + image: + repository: rancher/mirrored-library-nginx + tag: 1.19.2-alpine + + ## Containers allows injecting additional containers. This is meant to allow adding an authentication proxy to a Prometheus pod. + ## if using proxy extraContainer update targetPort with proxy container port + containers: | + - name: prometheus-proxy + args: + - nginx + - -g + - daemon off; + - -c + - /nginx/nginx.conf + image: "{{ template "system_default_registry" . }}{{ .Values.prometheus.prometheusSpec.proxy.image.repository }}:{{ .Values.prometheus.prometheusSpec.proxy.image.tag }}" + ports: + - containerPort: 8080 + name: nginx-http + protocol: TCP + volumeMounts: + - mountPath: /nginx + name: prometheus-nginx + - mountPath: /var/cache/nginx + name: nginx-home + securityContext: + runAsUser: 101 + runAsGroup: 101 + + # Additional volumes on the output StatefulSet definition. + volumes: + - name: nginx-home + emptyDir: {} + - name: prometheus-nginx + configMap: + name: prometheus-nginx-proxy-config + defaultMode: 438 + + # Additional VolumeMounts on the output StatefulSet definition. + volumeMounts: [] + + ## InitContainers allows injecting additional initContainers. This is meant to allow doing some changes + ## (permissions, dir tree) on mounted volumes before starting prometheus + initContainers: [] + + ## PortName to use for Prometheus. + ## + portName: "nginx-http" + + additionalServiceMonitors: [] + ## Name of the ServiceMonitor to create + ## + # - name: "" + + ## Additional labels to set used for the ServiceMonitorSelector. Together with standard labels from + ## the chart + ## + # additionalLabels: {} + + ## Service label for use in assembling a job name of the form