From de67ba10c085ffa7e08cacb7360cd7d10136e8e8 Mon Sep 17 00:00:00 2001
From: Nik Afiq <nik.afiq98@ymail.com>
Date: Fri, 10 Apr 2026 19:09:54 +0900
Subject: [PATCH] feat: add metric relabeling to drop high-cardinality metrics
 from kube-state-metrics

---
 values/kube-prometheus-stack.yaml | 39 ++++++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)

diff --git a/values/kube-prometheus-stack.yaml b/values/kube-prometheus-stack.yaml
index d7b3ad2..33cc72a 100644
--- a/values/kube-prometheus-stack.yaml
+++ b/values/kube-prometheus-stack.yaml
@@ -75,6 +75,7 @@ prometheus:
   prometheusSpec:
     enableRemoteWriteReceiver: true
     retention: 15d
+    retentionSize: 8GB
     storageSpec:
       volumeClaimTemplate:
         metadata:
@@ -99,4 +100,40 @@ alertmanager:
             - ReadWriteOnce
           resources:
             requests:
-              storage: 1Gi
\ No newline at end of file
+              storage: 1Gi
+
+## Drop high-cardinality metrics from kube-state-metrics to reduce Prometheus load and storage requirements
+kubeApiServer:
+  serviceMonitor:
+    metricRelabelings:
+      - sourceLabels: [__name__]
+        regex: "apiserver_request_duration_seconds_bucket|apiserver_request_sli_duration_seconds_bucket|apiserver_request_body_size_bytes_bucket|apiserver_response_sizes_bucket|apiserver_watch_cache_read_wait_seconds_bucket|apiserver_watch_events_sizes_bucket"
+        action: drop
+
+kubeEtcd:
+  serviceMonitor:
+    metricRelabelings:
+      - sourceLabels: [__name__]
+        regex: "etcd_request_duration_seconds_bucket"
+        action: drop
+
+kubeScheduler:
+  serviceMonitor:
+    metricRelabelings:
+      - sourceLabels: [__name__]
+        regex: "scheduler_plugin_execution_duration_seconds_bucket"
+        action: drop
+
+kubeControllerManager:
+  serviceMonitor:
+    metricRelabelings:
+      - sourceLabels: [__name__]
+        regex: "workqueue_queue_duration_seconds_bucket|workqueue_work_duration_seconds_bucket"
+        action: drop
+
+kubelet:
+  serviceMonitor:
+    metricRelabelings:
+      - sourceLabels: [__name__]
+        regex: "kubelet_runtime_operations_duration_seconds_bucket|prober_probe_duration_seconds_bucket"
+        action: drop
\ No newline at end of file