Grafana実践 - 観測性ダッシュボードを構築する

Grafanaとは

Grafanaは、時系列データを可視化するためのオープンソース観測性プラットフォームです。Prometheus、Loki、Tempo、Elasticsearch、CloudWatchなど、多様なデータソースに対応しています。

主要機能

Grafanaのインストール

Docker Compose

# docker-compose.yml
services:
 grafana:
 image: grafana/grafana:11.0.0
 container_name: grafana
 ports:
 - "3000:3000"
 environment:
 - GF_SECURITY_ADMIN_USER=admin
 - GF_SECURITY_ADMIN_PASSWORD=YOUR_GRAFANA_ADMIN_PASSWORD
 - GF_USERS_ALLOW_SIGN_UP=false
 - GF_SERVER_ROOT_URL=http://localhost:3000
 - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-piechart-panel
 volumes:
 - grafana-storage:/var/lib/grafana
 - ./provisioning:/etc/grafana/provisioning
 networks:
 - monitoring

 prometheus:
 image: prom/prometheus:v2.50.0
 container_name: prometheus
 ports:
 - "9090:9090"
 volumes:
 - ./prometheus.yml:/etc/prometheus/prometheus.yml
 - prometheus-storage:/prometheus
 command:
 - '--config.file=/etc/prometheus/prometheus.yml'
 - '--storage.tsdb.path=/prometheus'
 networks:
 - monitoring

 loki:
 image: grafana/loki:2.9.0
 container_name: loki
 ports:
 - "3100:3100"
 volumes:
 - ./loki-config.yml:/etc/loki/local-config.yaml
 - loki-storage:/loki
 command: -config.file=/etc/loki/local-config.yaml
 networks:
 - monitoring

volumes:
 grafana-storage:
 prometheus-storage:
 loki-storage:

networks:
 monitoring:
 driver: bridge

# 起動
docker compose up -d

# http://localhost:3000 にアクセス
# ユーザー名: admin
# パスワード: admin

Kubernetes (Helm)

# Helmリポジトリ追加
helm repo add grafana https://grafana.github.io/helm-charts
helm repo update

# インストール
helm install grafana grafana/grafana \
 --namespace monitoring \
 --create-namespace \
 --set persistence.enabled=true \
 --set persistence.size=10Gi \
 --set adminPassword=YOUR_GRAFANA_ADMIN_PASSWORD

# パスワード取得
kubectl get secret --namespace monitoring grafana -o jsonpath="{.data.admin-password}" | base64 --decode

# ポートフォワード
kubectl port-forward -n monitoring svc/grafana 3000:80

データソースの設定

Prometheus

# provisioning/datasources/prometheus.yaml
apiVersion: 1

datasources:
 - name: Prometheus
 type: prometheus
 access: proxy
 url: http://prometheus:9090
 isDefault: true
 editable: true
 jsonData:
 httpMethod: POST
 exemplarTraceIdDestinations:
 - name: trace_id
 datasourceUid: tempo

Loki (ログ)

# provisioning/datasources/loki.yaml
apiVersion: 1

datasources:
 - name: Loki
 type: loki
 access: proxy
 url: http://loki:3100
 editable: true
 jsonData:
 derivedFields:
 - datasourceUid: tempo
 matcherRegex: "trace_id=(\\w+)"
 name: TraceID
 url: "$${__value.raw}"

Tempo (トレース)

# provisioning/datasources/tempo.yaml
apiVersion: 1

datasources:
 - name: Tempo
 type: tempo
 access: proxy
 url: http://tempo:3200
 editable: true
 jsonData:
 tracesToLogsV2:
 datasourceUid: loki
 spanStartTimeShift: '-1h'
 spanEndTimeShift: '1h'
 filterByTraceID: true
 filterBySpanID: false
 tracesToMetrics:
 datasourceUid: prometheus
 serviceMap:
 datasourceUid: prometheus
 nodeGraph:
 enabled: true

ダッシュボード作成

基本構造

{
 "dashboard": {
 "title": "Application Metrics",
 "tags": ["app", "production"],
 "timezone": "browser",
 "schemaVersion": 38,
 "version": 1,
 "refresh": "10s",
 "time": {
 "from": "now-1h",
 "to": "now"
 },
 "panels": [
 {
 "id": 1,
 "type": "timeseries",
 "title": "Request Rate",
 "gridPos": {
 "x": 0,
 "y": 0,
 "w": 12,
 "h": 8
 },
 "targets": [
 {
 "expr": "rate(http_requests_total[5m])",
 "legendFormat": "{{method}} {{path}}"
 }
 ],
 "fieldConfig": {
 "defaults": {
 "unit": "reqps",
 "color": {
 "mode": "palette-classic"
 }
 }
 }
 }
 ]
 }
}

PromQLクエリ例

# リクエストレート（RPS）
rate(http_requests_total[5m])

# P95レイテンシ
histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))

# エラー率
sum(rate(http_requests_total{status=~"5.."}[5m])) 
/ 
sum(rate(http_requests_total[5m]))

# CPU使用率
100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# メモリ使用率
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100

# ディスク使用率
(node_filesystem_size_bytes - node_filesystem_avail_bytes) 
/ 
node_filesystem_size_bytes * 100

# Podレプリカ数
kube_deployment_status_replicas{deployment="my-app"}

# コンテナ再起動回数
increase(kube_pod_container_status_restarts_total[1h])

Variables（変数）

{
 "templating": {
 "list": [
 {
 "name": "namespace",
 "type": "query",
 "datasource": "Prometheus",
 "query": "label_values(kube_pod_info, namespace)",
 "refresh": 1,
 "includeAll": true
 },
 {
 "name": "pod",
 "type": "query",
 "datasource": "Prometheus",
 "query": "label_values(kube_pod_info{namespace=\"$namespace\"}, pod)",
 "refresh": 2,
 "includeAll": true
 },
 {
 "name": "interval",
 "type": "interval",
 "query": "1m,5m,10m,30m,1h",
 "auto": true,
 "auto_count": 30,
 "auto_min": "10s"
 }
 ]
 }
}

パネルタイプ

ダッシュボード例

RED Method（Rate, Errors, Duration）

# provisioning/dashboards/red-dashboard.json
{
 "dashboard": {
 "title": "RED Metrics - My App",
 "panels": [
 {
 "id": 1,
 "title": "Request Rate",
 "type": "timeseries",
 "targets": [{
 "expr": "sum(rate(http_requests_total{app=\"my-app\"}[5m])) by (method)"
 }]
 },
 {
 "id": 2,
 "title": "Error Rate",
 "type": "timeseries",
 "targets": [{
 "expr": "sum(rate(http_requests_total{app=\"my-app\",status=~\"5..\"}[5m])) / sum(rate(http_requests_total{app=\"my-app\"}[5m]))"
 }],
 "fieldConfig": {
 "defaults": {
 "unit": "percentunit"
 }
 }
 },
 {
 "id": 3,
 "title": "Request Duration (P95)",
 "type": "timeseries",
 "targets": [{
 "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{app=\"my-app\"}[5m])) by (le))"
 }],
 "fieldConfig": {
 "defaults": {
 "unit": "s"
 }
 }
 }
 ]
 }
}

USE Method（Utilization, Saturation, Errors）

# CPU Utilization
avg(rate(container_cpu_usage_seconds_total{pod=~"my-app.*"}[5m])) by (pod)

# Memory Utilization
avg(container_memory_working_set_bytes{pod=~"my-app.*"}) by (pod)

# Network Saturation
rate(container_network_transmit_bytes_total{pod=~"my-app.*"}[5m])

# Disk I/O Errors
rate(node_disk_io_now{device="sda"}[5m])

LogQLクエリ（Loki）

# 特定アプリのログ
{app="my-app"}

# エラーログのみ
{app="my-app"} |= "error"

# JSON解析
{app="my-app"} | json | level="error"

# レート計算
rate({app="my-app"} |= "error" [5m])

# トレースIDでフィルタ
{app="my-app"} | json | trace_id="abc123"

# ステータスコード500のカウント
sum(count_over_time({app="my-app"} | json | status="500" [1h]))

アラート設定

Alerting Rule

# provisioning/alerting/alert-rules.yaml
apiVersion: 1

groups:
 - name: my-app-alerts
 folder: Application
 interval: 1m
 rules:
 - uid: high-error-rate
 title: High Error Rate
 condition: C
 data:
 - refId: A
 relativeTimeRange:
 from: 600
 to: 0
 datasourceUid: prometheus
 model:
 expr: |
 sum(rate(http_requests_total{status=~"5.."}[5m])) 
 / 
 sum(rate(http_requests_total[5m]))
 - refId: C
 datasourceUid: __expr__
 model:
 type: threshold
 expression: A
 conditions:
 - evaluator:
 params: [0.05]
 type: gt
 operator:
 type: and
 noDataState: NoData
 execErrState: Alerting
 for: 5m
 annotations:
 summary: Error rate is above 5%
 labels:
 severity: critical
 team: platform

Contact Points

# provisioning/alerting/contact-points.yaml
apiVersion: 1

contactPoints:
 - orgId: 1
 name: slack-platform
 receivers:
 - uid: slack-1
 type: slack
 settings:
 url: https://hooks.slack.com/services/YOUR/WEBHOOK/URL
 text: |
 {{ range .Alerts }}
 *{{ .Labels.alertname }}*
 {{ .Annotations.summary }}
 {{ end }}

Notification Policies

# provisioning/alerting/notification-policies.yaml
apiVersion: 1

policies:
 - orgId: 1
 receiver: slack-platform
 group_by: ['alertname', 'severity']
 group_wait: 10s
 group_interval: 5m
 repeat_interval: 12h
 routes:
 - receiver: slack-platform
 matchers:
 - severity = critical
 continue: true
 - receiver: pagerduty
 matchers:
 - severity = critical
 - team = platform

Provisioningによるコード管理

ディレクトリ構造

provisioning/
├── datasources/
│ ├── prometheus.yaml
│ ├── loki.yaml
│ └── tempo.yaml
├── dashboards/
│ ├── dashboards.yaml # ダッシュボードプロバイダ設定
│ └── app-metrics.json # 実際のダッシュボード定義
├── alerting/
│ ├── alert-rules.yaml
│ ├── contact-points.yaml
│ └── notification-policies.yaml
└── notifiers/
 └── slack.yaml

ダッシュボードプロバイダ

# provisioning/dashboards/dashboards.yaml
apiVersion: 1

providers:
 - name: 'default'
 orgId: 1
 folder: 'General'
 type: file
 disableDeletion: false
 updateIntervalSeconds: 10
 allowUiUpdates: true
 options:
 path: /etc/grafana/provisioning/dashboards
 foldersFromFilesStructure: true

エクスポートとインポート

# ダッシュボードのエクスポート（API経由）
curl -H "Authorization: Bearer ${GRAFANA_API_KEY}" \
 http://localhost:3000/api/dashboards/uid/abc123 \
 | jq '.dashboard' > dashboard.json

# ダッシュボードのインポート
curl -X POST \
 -H "Content-Type: application/json" \
 -H "Authorization: Bearer ${GRAFANA_API_KEY}" \
 -d @dashboard.json \
 http://localhost:3000/api/dashboards/db

Grafanaの高度な機能

Trace to Metrics（トレース→メトリクス）

# Tempoデータソース設定
jsonData:
 tracesToMetrics:
 datasourceUid: prometheus
 queries:
 - name: 'Request rate'
 query: 'sum(rate(tempo_spanmetrics_calls_total{$$__tags}[5m]))'
 - name: 'Error rate'
 query: 'sum(rate(tempo_spanmetrics_calls_total{status_code="STATUS_CODE_ERROR",$$__tags}[5m])) / sum(rate(tempo_spanmetrics_calls_total{$$__tags}[5m]))'

Logs to Traces（ログ→トレース）

# Lokiデータソース設定
jsonData:
 derivedFields:
 - datasourceUid: tempo
 matcherRegex: "trace_id=(\\w+)"
 name: TraceID
 url: "$${__value.raw}"

Exemplars（サンプル）

# Prometheusクエリ with exemplars
histogram_quantile(0.95, 
 sum(rate(http_request_duration_seconds_bucket[5m])) by (le)
)

パフォーマンス最適化

# grafana.ini
[dashboards]
# バージョン管理の保持数
versions_to_keep = 20

[dataproxy]
# データソースタイムアウト
timeout = 30

[query]
# クエリキャッシュ
cache_ttl = 5m

[explore]
# Exploreモードのログ行数制限
max_lines = 1000

まとめ

この記事では、手順を追って動かしながら重要な確認ポイントを押さえることが大切です。まずは学習用の最小構成で動作確認し、本番利用する前にセキュリティ、権限、エラー処理、運用手順を見直してください。

参考リソース

• 公式ドキュメント - Grafana実践 - 観測性ダッシュボードを構築する を確認するための一次情報

関連記事

• Prometheusモニタリング実践 - メトリクス収集とアラート
• OpenTelemetry で分散トレーシング計装

×

PR

×

PR