helm repo add prometheus-community <https://prometheus-community.github.io/helm-charts>
helm repo update

# prometheus-values.yaml
prometheus:
  service:
    type: ClusterIP
grafana:
  service:
    type: ClusterIP

helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack -f custom-values.yaml -n prometheus-monitoring

# verify the services
kubectl get services

kubectl get services -n prometheus-monitoring

# Output
NAME                                             TYPE           CLUSTER-IP      EXTERNAL-IP       PORT(S)                         AGE
alertmanager-operated                            ClusterIP      None            <none>            9093/TCP,9094/TCP,9094/UDP      21m
kube-prometheus-stack-alertmanager               ClusterIP      10.233.63.160   <none>            9093/TCP,8080/TCP               21m
kube-prometheus-stack-grafana                    LoadBalancer   10.233.25.34    <none>            80:32258/TCP                    21m
kube-prometheus-stack-kube-state-metrics         ClusterIP      10.233.43.75    <none>            8080/TCP                        21m
kube-prometheus-stack-operator                   ClusterIP      10.233.27.200   <none>            443/TCP                         21m
kube-prometheus-stack-prometheus                 LoadBalancer   10.233.60.168   <none>            9090:32377/TCP,8080:32587/TCP   21m
kube-prometheus-stack-prometheus-node-exporter   ClusterIP      10.233.21.240   <none>            9100/TCP                        21m
prometheus-operated                              ClusterIP      None            <none>            9090/TCP                        21m

Create HTTPRoute so Traefik can redirect the request to the desired services:

# grafana-httproutes.yaml

apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: grafana-httproute
  namespace: prometheus-monitoring
spec:
  parentRefs:
  - name: traefik
    sectionName: web
    namespace: default
  hostnames:
  - grafana.kube.slc.net
  rules:
  - filters:
    - type: RequestRedirect
      requestRedirect:
        scheme: https
        statusCode: 301

---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  labels:
    gateway: traefik
  name: grafana-httpsroute
  namespace: prometheus-monitoring
spec:
  hostnames:
  - grafana.kube.slc.net

  parentRefs:
  - group: gateway.networking.k8s.io
		kind: Gateway
    name: traefik
    namespace: default
    sectionName: websecure

  rules:
  - backendRefs:
    - group: ""
      kind: Service
      name: kube-prometheus-stack-grafana
      namespace: prometheus-monitoring
      port: 80
      weight: 1

    matches:
      - path:
          type: PathPrefix
          value: /

# prometheus-httproutes.yaml
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: prometheus-httproute
  namespace: prometheus-monitoring
spec:
  parentRefs:
  - name: traefik
    sectionName: web
    namespace: default
  hostnames:
  - prometheus.kube.slc.net
  rules:
  - filters:
    - type: RequestRedirect
      requestRedirect:
        scheme: https
        statusCode: 301

---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  labels:
    gateway: traefik
  name: prometheus-httpsroute
  namespace: prometheus-monitoring
spec:
  hostnames:
  - prometheus.kube.slc.net

  parentRefs:
  - group: gateway.networking.k8s.io
		kind: Gateway
    name: traefik
    namespace: default
    sectionName: websecure

  rules:
  - backendRefs:
    - group: ""
      kind: Service
      name: kube-prometheus-stack-prometheus
      namespace: prometheus-monitoring
      port: 80
      weight: 1

    matches:
      - path:
          type: PathPrefix
          value: /

Cara dapet default user grafana login:

# Get Username
kubectl get secret --namespace default kube-prometheus-stack-grafana -o jsonpath="{.data.admin-user}" | base64 --decode ; echo

# Get Password
kubectl get secret --namespace default kube-prometheus-stack-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo

Dah sisa kita ngeset buat monitoring persistent volumes:

Buka hamburger menu di kiri atas > Dashboards

Klik New > New Dashboard

Pilih yang Import Dashboard, masukin angka 11455