Add monitoring setup with Grafana, Loki, and Prometheus configurations

.env.example

@@ -6,4 +6,7 @@ PORKBUN_SECRET_KEY=sk1_your_key_here
 K3S_NODE_TOKEN=your_token_here
 
 # Gitea runner token for CI/CD
-GITEA_RUNNER_TOKEN=your_token_here
+GITEA_RUNNER_TOKEN=your_token_here
+
+# Grafana admin password
+GRAFANA_ADMIN_PASSWORD=your_password_here

ansible/playbooks/setup-monitoring.yaml

@@ -0,0 +1,4 @@
+- name: Prepare monitoring storage on Minisforum
+  hosts: minisforum
+  roles:
+    - monitoring

ansible/roles/monitoring/tasks/main.yaml

@@ -0,0 +1,11 @@
+- name: Create monitoring data directories
+  file:
+    path: "{{ item.path }}"
+    state: directory
+    owner: "{{ item.owner }}"
+    group: "{{ item.owner }}"
+    mode: "0755"
+  loop:
+    - { path: /data/prometheus, owner: "65534" }  # nobody — Prometheus UID
+    - { path: /data/loki,       owner: "10001"  }  # Loki UID
+  become: true

manifests/grafana-loki-datasource.yaml

@@ -0,0 +1,20 @@
+# Grafana Loki datasource configuration for Grafana in the monitoring namespace
+# Apply: kubectl apply -f manifests/grafana-loki-datasource.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-loki-datasource
+  namespace: monitoring
+  labels:
+    grafana_datasource: "1"
+data:
+  loki-datasource.yaml: |
+    apiVersion: 1
+    datasources:
+      - name: Loki
+        type: loki
+        access: proxy
+        url: http://loki-stack.monitoring.svc.cluster.local:3100
+        isDefault: false
+        version: 1
+        editable: true

manifests/grafana-secret.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -euo pipefail
+source "$(dirname "$0")/../.env"
+
+kubectl create secret generic grafana-admin-secret \
+  --namespace monitoring \
+  --from-literal=admin-password="${GRAFANA_ADMIN_PASSWORD}" \
+  --from-literal=admin-user="admin" \
+  --dry-run=client -o yaml | kubectl apply -f -

manifests/monitoring-pvs.yaml

@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: prometheus-pv
+spec:
+  capacity:
+    storage: 20Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: ""
+  hostPath:
+    path: /data/prometheus
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: grafana-pv
+spec:
+  capacity:
+    storage: 2Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: ""
+  hostPath:
+    path: /data/grafana
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: loki-pv
+spec:
+  capacity:
+    storage: 10Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: ""
+  hostPath:
+    path: /data/loki

values/kube-prometheus-stack.yaml

@@ -0,0 +1,61 @@
+# kube-prometheus-stack
+# Chart: 82.10.2 / App: v0.89.0
+#
+# helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack \
+#   --namespace monitoring --create-namespace \
+#   -f values/kube-prometheus-stack.yaml
+
+grafana:
+  admin:
+    existingSecret: grafana-admin-secret
+    userKey: admin-user
+    passwordKey: admin-password
+
+  ingress:
+    enabled: true
+    ingressClassName: traefik
+    annotations:
+      traefik.ingress.kubernetes.io/router.entrypoints: websecure
+      traefik.ingress.kubernetes.io/router.tls: "true"
+      cert-manager.io/cluster-issuer: letsencrypt-prod
+    hosts:
+      - grafana.nik4nao.com
+    tls:
+      - secretName: grafana-tls
+        hosts:
+          - grafana.nik4nao.com
+
+  persistence:
+    enabled: true
+    size: 2Gi
+    initChownData: true
+
+prometheus:
+  prometheusSpec:
+    retention: 15d
+
+    storageSpec:
+      volumeClaimTemplate:
+        metadata:
+          annotations:
+            helm.sh/resource-policy: keep
+        spec:
+          storageClassName: ""
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 20Gi
+          volumeName: prometheus-pv
+
+alertmanager:
+  alertmanagerSpec:
+    storage:
+      volumeClaimTemplate:
+        spec:
+          storageClassName: local-path
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 1Gi

values/loki-stack.yaml

@@ -0,0 +1,23 @@
+# loki-stack (Loki + Promtail)
+# Chart: 2.10.3 / App: v2.9.3
+#
+# helm upgrade --install loki-stack grafana/loki-stack \
+#   --namespace monitoring --create-namespace \
+#   -f values/loki-stack.yaml
+
+loki:
+  persistence:
+    enabled: true
+    storageClassName: ""
+    accessModes:
+      - ReadWriteOnce
+    size: 10Gi
+    volumeName: loki-pv
+  annotations:
+    helm.sh/resource-policy: keep
+
+promtail:
+  enabled: true
+
+grafana:
+  enabled: false

values/traefik.yaml

@@ -35,6 +35,7 @@ ports:
           to: websecure
           scheme: https
           permanent: true
+          priority: 1
   websecure:
     port: 8443
     exposedPort: 443