feat: add metric relabeling to drop high-cardinality metrics from kube-state-metrics

values/kube-prometheus-stack.yaml

@@ -75,6 +75,7 @@ prometheus:
   prometheusSpec:
     enableRemoteWriteReceiver: true
     retention: 15d
+    retentionSize: 8GB
     storageSpec:
       volumeClaimTemplate:
         metadata:
@@ -100,3 +101,39 @@ alertmanager:
           resources:
             requests:
               storage: 1Gi
+
+## Drop high-cardinality metrics from kube-state-metrics to reduce Prometheus load and storage requirements
+kubeApiServer:
+  serviceMonitor:
+    metricRelabelings:
+      - sourceLabels: [__name__]
+        regex: "apiserver_request_duration_seconds_bucket|apiserver_request_sli_duration_seconds_bucket|apiserver_request_body_size_bytes_bucket|apiserver_response_sizes_bucket|apiserver_watch_cache_read_wait_seconds_bucket|apiserver_watch_events_sizes_bucket"
+        action: drop
+
+kubeEtcd:
+  serviceMonitor:
+    metricRelabelings:
+      - sourceLabels: [__name__]
+        regex: "etcd_request_duration_seconds_bucket"
+        action: drop
+
+kubeScheduler:
+  serviceMonitor:
+    metricRelabelings:
+      - sourceLabels: [__name__]
+        regex: "scheduler_plugin_execution_duration_seconds_bucket"
+        action: drop
+
+kubeControllerManager:
+  serviceMonitor:
+    metricRelabelings:
+      - sourceLabels: [__name__]
+        regex: "workqueue_queue_duration_seconds_bucket|workqueue_work_duration_seconds_bucket"
+        action: drop
+
+kubelet:
+  serviceMonitor:
+    metricRelabelings:
+      - sourceLabels: [__name__]
+        regex: "kubelet_runtime_operations_duration_seconds_bucket|prober_probe_duration_seconds_bucket"
+        action: drop