Kubernetes 모니터링 — Prometheus와 Grafana로 클러스터 상태를 파악하는 방법

# 설치
kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml

# 사용
kubectl top nodes
# NAME     CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%
# node-1   350m         17%    2048Mi          51%

kubectl top pods -n production
# NAME        CPU(cores)   MEMORY(bytes)
# web-abc     120m         256Mi

대상(Pod, Node) ←── 스크레이핑 ── Prometheus Server ──→ Grafana
       ↑                              ↓
  /metrics 엔드포인트          AlertManager ──→ Slack, PagerDuty

# Helm으로 한 번에 설치 (Prometheus + Grafana + AlertManager + Exporters)
helm install monitoring prometheus-community/kube-prometheus-stack \
  --namespace monitoring \
  --create-namespace \
  --set grafana.adminPassword=admin

# Pod 상태 메트릭
kube_pod_status_phase{namespace="production",pod="web-abc",phase="Running"} 1
kube_pod_container_status_restarts_total{namespace="production",pod="web-abc"} 3

# Deployment 상태
kube_deployment_status_replicas_available{deployment="web"} 3
kube_deployment_spec_replicas{deployment="web"} 3

# 리소스 요청/제한
kube_pod_container_resource_requests{resource="cpu"} 0.2
kube_pod_container_resource_limits{resource="memory"} 536870912

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: my-app-monitor
  namespace: monitoring
  labels:
    release: monitoring    # Prometheus가 이 레이블로 ServiceMonitor를 찾음
spec:
  namespaceSelector:
    matchNames:
      - production
  selector:
    matchLabels:
      app: my-app
  endpoints:
    - port: metrics        # Service의 포트 이름
      interval: 15s
      path: /metrics

# Prometheus 메트릭 형식 예제
# HELP http_requests_total Total number of HTTP requests
# TYPE http_requests_total counter
http_requests_total{method="GET",status="200"} 12345
http_requests_total{method="POST",status="201"} 678

# HELP http_request_duration_seconds HTTP request latency
# TYPE http_request_duration_seconds histogram
http_request_duration_seconds_bucket{le="0.1"} 8000
http_request_duration_seconds_bucket{le="0.5"} 11000
http_request_duration_seconds_bucket{le="1.0"} 12000

# 네임스페이스별 CPU 사용률
sum(rate(container_cpu_usage_seconds_total{namespace="production"}[5m])) by (pod)

# CPU 쓰로틀링 비율
sum(rate(container_cpu_cfs_throttled_periods_total[5m]))
/ sum(rate(container_cpu_cfs_periods_total[5m]))

# 메모리 사용량 (limit 대비)
container_memory_working_set_bytes{namespace="production"}
/ container_spec_memory_limit_bytes{namespace="production"}

# OOMKill 발생 횟수
kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}

# 재시작 횟수가 많은 Pod
kube_pod_container_status_restarts_total > 5

# Running이 아닌 Pod
kube_pod_status_phase{phase!="Running",phase!="Succeeded"} == 1

# Pending 상태 Pod
kube_pod_status_phase{phase="Pending"} == 1

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: pod-alerts
  namespace: monitoring
  labels:
    release: monitoring
spec:
  groups:
    - name: pod-health
      rules:
        - alert: PodCrashLooping
          expr: rate(kube_pod_container_status_restarts_total[15m]) > 0.1
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Pod {{ $labels.pod }}이 반복적으로 재시작 중"
            description: "15분간 재시작 비율이 0.1을 초과했습니다"

        - alert: HighMemoryUsage
          expr: |
            container_memory_working_set_bytes / container_spec_memory_limit_bytes > 0.9
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "{{ $labels.pod }}의 메모리 사용률이 90%를 초과"

# Grafana 접근 (포트포워딩)
kubectl port-forward svc/monitoring-grafana -n monitoring 3000:80

# 브라우저에서 http://localhost:3000 접속
# 기본 계정: admin / admin (또는 설정한 비밀번호)

[즉시 대응]
- Pod CrashLoopBackOff
- 노드 NotReady
- 디스크 사용률 > 90%

[주의]
- CPU 쓰로틀링 비율 > 25%
- 메모리 사용률 > 85%
- HPA가 maxReplicas에 도달