{
   "description": "Controller Runtime",
   "graphTooltip": 1,
   "panels": [
      {
         "datasource": {
            "type": "datasource",
            "uid": "-- Mixed --"
         },
         "fieldConfig": {
            "defaults": {
               "decimals": 0,
               "unit": null
            }
         },
         "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 0
         },
         "id": 1,
         "pluginVersion": "v11.0.0",
         "targets": [
            {
               "datasource": {
                  "type": "prometheus",
                  "uid": "prometheus"
               },
               "expr": "controller_runtime_active_workers{job=\"$job\", namespace=\"$namespace\"}",
               "legendFormat": "{{controller}} {{instance}}"
            }
         ],
         "title": "Number of Workers in Use",
         "type": "stat"
      },
      {
         "datasource": {
            "type": "datasource",
            "uid": "-- Mixed --"
         },
         "fieldConfig": {
            "defaults": {
               "decimals": null,
               "unit": null
            }
         },
         "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 8
         },
         "id": 2,
         "pluginVersion": "v11.0.0",
         "targets": [
            {
               "datasource": {
                  "type": "prometheus",
                  "uid": "prometheus"
               },
               "expr": "sum(rate(controller_runtime_reconcile_errors_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)",
               "legendFormat": "{{instance}} {{pod}}"
            }
         ],
         "title": "Reconciliation Error Count per Controller",
         "type": "timeseries"
      },
      {
         "datasource": {
            "type": "datasource",
            "uid": "-- Mixed --"
         },
         "fieldConfig": {
            "defaults": {
               "decimals": null,
               "unit": null
            }
         },
         "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 16
         },
         "id": 3,
         "pluginVersion": "v11.0.0",
         "targets": [
            {
               "datasource": {
                  "type": "prometheus",
                  "uid": "prometheus"
               },
               "expr": "sum(rate(controller_runtime_reconcile_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, pod)",
               "legendFormat": "{{instance}} {{pod}}"
            }
         ],
         "title": "Total Reconciliation Count per Controller",
         "type": "timeseries"
      },
      {
         "datasource": {
            "type": "datasource",
            "uid": "-- Mixed --"
         },
         "fieldConfig": {
            "defaults": {
               "decimals": 0,
               "unit": null
            }
         },
         "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 24
         },
         "id": 4,
         "pluginVersion": "v11.0.0",
         "targets": [
            {
               "datasource": {
                  "type": "prometheus",
                  "uid": "prometheus"
               },
               "expr": "workqueue_depth{job=\"$job\", namespace=\"$namespace\"}",
               "legendFormat": "{{instance}} {{pod}}"
            }
         ],
         "title": "WorkQueue Depth",
         "type": "stat"
      },
      {
         "datasource": {
            "type": "datasource",
            "uid": "-- Mixed --"
         },
         "fieldConfig": {
            "defaults": {
               "decimals": null,
               "unit": null
            }
         },
         "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 32
         },
         "id": 5,
         "pluginVersion": "v11.0.0",
         "targets": [
            {
               "datasource": {
                  "type": "prometheus",
                  "uid": "prometheus"
               },
               "expr": "histogram_quantile(0.50, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
               "legendFormat": "P50 {{name}}"
            }
         ],
         "title": "Seconds for Items Stay in Queue (before being requested) P50",
         "type": "timeseries"
      },
      {
         "datasource": {
            "type": "datasource",
            "uid": "-- Mixed --"
         },
         "fieldConfig": {
            "defaults": {
               "decimals": null,
               "unit": null
            }
         },
         "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 40
         },
         "id": 6,
         "pluginVersion": "v11.0.0",
         "targets": [
            {
               "datasource": {
                  "type": "prometheus",
                  "uid": "prometheus"
               },
               "expr": "histogram_quantile(0.90, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
               "legendFormat": "P90 {{name}}"
            }
         ],
         "title": "Seconds for Items Stay in Queue (before being requested) P90",
         "type": "timeseries"
      },
      {
         "datasource": {
            "type": "datasource",
            "uid": "-- Mixed --"
         },
         "fieldConfig": {
            "defaults": {
               "decimals": null,
               "unit": null
            }
         },
         "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 48
         },
         "id": 7,
         "pluginVersion": "v11.0.0",
         "targets": [
            {
               "datasource": {
                  "type": "prometheus",
                  "uid": "prometheus"
               },
               "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
               "legendFormat": "P99 {{name}}"
            }
         ],
         "title": "Seconds for Items Stay in Queue (before being requested) P99",
         "type": "timeseries"
      },
      {
         "datasource": {
            "type": "datasource",
            "uid": "-- Mixed --"
         },
         "fieldConfig": {
            "defaults": {
               "decimals": null,
               "unit": null
            }
         },
         "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 56
         },
         "id": 8,
         "pluginVersion": "v11.0.0",
         "targets": [
            {
               "datasource": {
                  "type": "prometheus",
                  "uid": "prometheus"
               },
               "expr": "sum(rate(workqueue_adds_total{job=\"$job\", namespace=\"$namespace\"}[2m])) by (instance, name)",
               "legendFormat": "{{name}} {{instance}}"
            }
         ],
         "title": "Work Queue Add Rate",
         "type": "timeseries"
      },
      {
         "datasource": {
            "type": "datasource",
            "uid": "-- Mixed --"
         },
         "fieldConfig": {
            "defaults": {
               "decimals": null,
               "unit": null
            }
         },
         "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 64
         },
         "id": 9,
         "pluginVersion": "v11.0.0",
         "targets": [
            {
               "datasource": {
                  "type": "prometheus",
                  "uid": "prometheus"
               },
               "expr": "rate(workqueue_unfinished_work_seconds{job=\"$job\", namespace=\"$namespace\"}[5m])",
               "legendFormat": "{{name}} {{instance}}"
            }
         ],
         "title": "Unfinished Seconds",
         "type": "stat"
      },
      {
         "datasource": {
            "type": "datasource",
            "uid": "-- Mixed --"
         },
         "fieldConfig": {
            "defaults": {
               "decimals": null,
               "unit": null
            }
         },
         "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 72
         },
         "id": 10,
         "pluginVersion": "v11.0.0",
         "targets": [
            {
               "datasource": {
                  "type": "prometheus",
                  "uid": "prometheus"
               },
               "expr": "histogram_quantile(0.50, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
               "legendFormat": "P50 {{name}}"
            }
         ],
         "title": "Seconds Processing Items from WorkQueue - 50th Percentile",
         "type": "timeseries"
      },
      {
         "datasource": {
            "type": "datasource",
            "uid": "-- Mixed --"
         },
         "fieldConfig": {
            "defaults": {
               "decimals": null,
               "unit": null
            }
         },
         "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 80
         },
         "id": 11,
         "pluginVersion": "v11.0.0",
         "targets": [
            {
               "datasource": {
                  "type": "prometheus",
                  "uid": "prometheus"
               },
               "expr": "histogram_quantile(0.90, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
               "legendFormat": "P90 {{name}}"
            }
         ],
         "title": "Seconds Processing Items from WorkQueue - 90th Percentile",
         "type": "timeseries"
      },
      {
         "datasource": {
            "type": "datasource",
            "uid": "-- Mixed --"
         },
         "fieldConfig": {
            "defaults": {
               "decimals": null,
               "unit": null
            }
         },
         "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 88
         },
         "id": 12,
         "pluginVersion": "v11.0.0",
         "targets": [
            {
               "datasource": {
                  "type": "prometheus",
                  "uid": "prometheus"
               },
               "expr": "histogram_quantile(0.99, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
               "legendFormat": "P99 {{name}}"
            }
         ],
         "title": "Seconds Processing Items from WorkQueue - 99th Percentile",
         "type": "timeseries"
      },
      {
         "datasource": {
            "type": "datasource",
            "uid": "-- Mixed --"
         },
         "fieldConfig": {
            "defaults": {
               "decimals": null,
               "unit": null
            }
         },
         "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 96
         },
         "id": 13,
         "pluginVersion": "v11.0.0",
         "targets": [
            {
               "datasource": {
                  "type": "prometheus",
                  "uid": "prometheus"
               },
               "expr": "sum(rate(workqueue_retries_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name)",
               "legendFormat": "{{name}} {{instance}}"
            }
         ],
         "title": "Work Queue Retries Rate",
         "type": "timeseries"
      }
   ],
   "schemaVersion": 39,
   "templating": {
      "list": [
         {
            "datasource": {
               "type": "prometheus",
               "uid": "prometheus"
            },
            "name": "namespace",
            "query": "label_values(controller_runtime_reconcile_total, namespace)",
            "refresh": 2,
            "type": "query"
         },
         {
            "datasource": {
               "type": "prometheus",
               "uid": "prometheus"
            },
            "name": "job",
            "query": "label_values(controller_runtime_reconcile_total{namespace=~\"$namespace\"}, job)",
            "refresh": 2,
            "type": "query"
         }
      ]
   },
   "time": {
      "from": "now-1h",
      "to": "now"
   },
   "timezone": "utc",
   "title": "Fleet / Controller-Runtime",
   "uid": "fleet-controller-runtime"
}