# Prometheus
firewall-cmd --permanent --add-port=9090/tcp

# Grafana
firewall-cmd --permanent --add-port=3000/tcp

# Alertmanager
firewall-cmd --permanent --add-port=9093/tcp

# Node Exporter
firewall-cmd --permanent --add-port=9100/tcp

# 重新載入
firewall-cmd --reload

# 1. 建立使用者
sudo useradd --no-create-home --shell /bin/false prometheus

# 2. 建立目錄
sudo mkdir -p /etc/prometheus
sudo mkdir -p /var/lib/prometheus

# 3. 下載 Prometheus（請確認最新版本）
cd /tmp
wget https://github.com/prometheus/prometheus/releases/download/v2.48.0/prometheus-2.48.0.linux-amd64.tar.gz

# 4. 解壓縮
tar xvfz prometheus-2.48.0.linux-amd64.tar.gz
cd prometheus-2.48.0.linux-amd64

# 5. 複製執行檔
sudo cp prometheus /usr/local/bin/
sudo cp promtool /usr/local/bin/

# 6. 複製設定檔
sudo cp -r consoles /etc/prometheus
sudo cp -r console_libraries /etc/prometheus
sudo cp prometheus.yml /etc/prometheus/

# 7. 設定權限
sudo chown -R prometheus:prometheus /etc/prometheus
sudo chown -R prometheus:prometheus /var/lib/prometheus
sudo chown prometheus:prometheus /usr/local/bin/prometheus
sudo chown prometheus:prometheus /usr/local/bin/promtool

sudo cat > /etc/systemd/system/prometheus.service << 'EOF'
[Unit]
Description=Prometheus
Wants=network-online.target
After=network-online.target

[Service]
User=prometheus
Group=prometheus
Type=simple
ExecStart=/usr/local/bin/prometheus \
    --config.file=/etc/prometheus/prometheus.yml \
    --storage.tsdb.path=/var/lib/prometheus/ \
    --storage.tsdb.retention.time=15d \
    --web.console.templates=/etc/prometheus/consoles \
    --web.console.libraries=/etc/prometheus/console_libraries \
    --web.enable-lifecycle \
    --web.enable-admin-api

ExecReload=/bin/kill -HUP $MAINPID
Restart=on-failure
RestartSec=5s

[Install]
WantedBy=multi-user.target
EOF

# 啟動服務
sudo systemctl daemon-reload
sudo systemctl enable prometheus
sudo systemctl start prometheus
sudo systemctl status prometheus

# 建立資料目錄
mkdir -p /data/prometheus

# 建立設定檔
cat > /data/prometheus/prometheus.yml << 'EOF'
global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']
EOF

# 執行容器
docker run -d \
  --name prometheus \
  --restart unless-stopped \
  -p 9090:9090 \
  -v /data/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml \
  -v /data/prometheus/data:/prometheus \
  prom/prometheus:v2.48.0 \
  --config.file=/etc/prometheus/prometheus.yml \
  --storage.tsdb.path=/prometheus \
  --storage.tsdb.retention.time=15d \
  --web.enable-lifecycle

# 檢查服務狀態
curl http://localhost:9090/-/healthy
# 預期回應: Prometheus Server is Healthy.

# 檢查設定
curl http://localhost:9090/api/v1/status/config

# 開啟 Web UI
# http://<server-ip>:9090

# CentOS/RHEL
cat > /etc/yum.repos.d/grafana.repo << 'EOF'
[grafana]
name=grafana
baseurl=https://rpm.grafana.com
repo_gpgcheck=1
enabled=1
gpgcheck=1
gpgkey=https://rpm.grafana.com/gpg.key
sslverify=1
sslcacert=/etc/pki/tls/certs/ca-bundle.crt
EOF

sudo yum install grafana -y

# Ubuntu/Debian
sudo apt-get install -y apt-transport-https software-properties-common
wget -q -O - https://apt.grafana.com/gpg.key | sudo apt-key add -
echo "deb https://apt.grafana.com stable main" | sudo tee /etc/apt/sources.list.d/grafana.list
sudo apt-get update
sudo apt-get install grafana -y

# 啟動服務
sudo systemctl daemon-reload
sudo systemctl enable grafana-server
sudo systemctl start grafana-server

# 建立資料目錄
mkdir -p /data/grafana

# 設定權限（Grafana 容器使用 UID 472）
sudo chown -R 472:472 /data/grafana

# 執行容器
docker run -d \
  --name grafana \
  --restart unless-stopped \
  -p 3000:3000 \
  -v /data/grafana:/var/lib/grafana \
  -e "GF_SECURITY_ADMIN_PASSWORD=your_secure_password" \
  -e "GF_USERS_ALLOW_SIGN_UP=false" \
  grafana/grafana:10.2.0

# 檢查服務狀態
curl http://localhost:3000/api/health
# 預期回應: {"commit":"...","database":"ok","version":"10.2.0"}

# 開啟 Web UI
# http://<server-ip>:3000
# 預設帳號: admin / admin

# 下載
cd /tmp
wget https://github.com/prometheus/node_exporter/releases/download/v1.7.0/node_exporter-1.7.0.linux-amd64.tar.gz
tar xvfz node_exporter-1.7.0.linux-amd64.tar.gz

# 安裝
sudo cp node_exporter-1.7.0.linux-amd64/node_exporter /usr/local/bin/
sudo useradd --no-create-home --shell /bin/false node_exporter

# 建立 Service
sudo cat > /etc/systemd/system/node_exporter.service << 'EOF'
[Unit]
Description=Node Exporter
Wants=network-online.target
After=network-online.target

[Service]
User=node_exporter
Group=node_exporter
Type=simple
ExecStart=/usr/local/bin/node_exporter \
    --collector.systemd \
    --collector.processes

[Install]
WantedBy=multi-user.target
EOF

# 啟動
sudo systemctl daemon-reload
sudo systemctl enable node_exporter
sudo systemctl start node_exporter

# 驗證
curl http://localhost:9100/metrics | head -20

/etc/prometheus/                    # Prometheus 設定目錄
├── prometheus.yml                  # 主要設定檔
├── rules/                          # 告警規則目錄
│   ├── node_alerts.yml
│   └── app_alerts.yml
├── consoles/                       # Console 模板
└── console_libraries/              # Console 函式庫

/var/lib/prometheus/                # Prometheus 資料目錄
├── chunks_head/                    # 記憶體資料
├── wal/                            # Write-Ahead Log
└── <block_id>/                     # 壓縮後的資料區塊

/etc/grafana/                       # Grafana 設定目錄
├── grafana.ini                     # 主要設定檔
└── provisioning/                   # 自動配置目錄
    ├── datasources/
    ├── dashboards/
    └── notifiers/

/var/lib/grafana/                   # Grafana 資料目錄
├── grafana.db                      # SQLite 資料庫
├── plugins/                        # 外掛目錄
└── png/                            # 圖片快取

# 檢查 Prometheus 設定語法
promtool check config /etc/prometheus/prometheus.yml

# 檢查告警規則語法
promtool check rules /etc/prometheus/rules/*.yml

# 查看 Prometheus 日誌
journalctl -u prometheus -f

# 查看 Grafana 日誌
journalctl -u grafana-server -f

# 測試 Exporter 連通性
curl -v http://target:9100/metrics

# 全域設定
global:
  scrape_interval: 15s          # 預設抓取間隔
  evaluation_interval: 15s      # 規則評估間隔
  scrape_timeout: 10s           # 抓取逾時
  external_labels:              # 外部標籤（用於 Federation）
    environment: production
    region: taiwan

# Alertmanager 設定
alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - alertmanager1:9093
            - alertmanager2:9093
      timeout: 10s

# 規則檔案
rule_files:
  - "/etc/prometheus/rules/*.yml"

# 抓取設定
scrape_configs:
  # Prometheus 自身監控
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']
    
  # Node Exporter - 主機監控
  - job_name: 'node'
    static_configs:
      - targets:
          - 'server1:9100'
          - 'server2:9100'
          - 'server3:9100'
    relabel_configs:
      - source_labels: [__address__]
        regex: '(.*):\d+'
        target_label: instance
        replacement: '${1}'

  # JVM 應用程式監控
  - job_name: 'jvm-apps'
    metrics_path: /actuator/prometheus
    static_configs:
      - targets:
          - 'app1:8080'
          - 'app2:8080'
        labels:
          application: 'order-service'
          team: 'backend'

  # Kubernetes Service Discovery
  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        target_label: __address__

# 遠端寫入（選用）
remote_write:
  - url: "http://thanos-receive:19291/api/v1/receive"
    queue_config:
      capacity: 10000
      max_shards: 30

scrape_configs:
  - job_name: 'secure-endpoint'
    scheme: https
    tls_config:
      ca_file: /etc/prometheus/ca.crt
      cert_file: /etc/prometheus/client.crt
      key_file: /etc/prometheus/client.key
      insecure_skip_verify: false
    basic_auth:
      username: prometheus
      password_file: /etc/prometheus/password
    static_configs:
      - targets: ['secure-server:443']

  - job_name: 'blackbox-http'
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
          - https://api.example.com/health
          - https://web.example.com
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115

# ✅ 好的設計
labels:
  env: production
  team: backend
  app: order-service

# ❌ 避免的設計
labels:
  request_id: abc123        # 高 Cardinality
  timestamp: 2024-01-01     # 高 Cardinality
  user_id: 12345            # 高 Cardinality

# 命令列參數
--storage.tsdb.retention.time=15d       # 資料保留天數
--storage.tsdb.retention.size=50GB      # 資料保留大小上限
--storage.tsdb.wal-compression          # 啟用 WAL 壓縮
--query.max-concurrency=20              # 最大並行查詢數
--query.timeout=2m                      # 查詢逾時

# /etc/grafana/provisioning/datasources/prometheus.yml
apiVersion: 1

datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: false
    jsonData:
      timeInterval: "15s"
      httpMethod: POST
      exemplarTraceIdDestinations:
        - name: traceID
          datasourceUid: jaeger

  - name: Prometheus-DR
    type: prometheus
    access: proxy
    url: http://prometheus-dr:9090
    editable: false

{
  "templating": {
    "list": [
      {
        "name": "environment",
        "type": "custom",
        "options": [
          {"text": "Production", "value": "prod"},
          {"text": "Staging", "value": "stage"},
          {"text": "Development", "value": "dev"}
        ],
        "current": {"text": "Production", "value": "prod"}
      },
      {
        "name": "instance",
        "type": "query",
        "datasource": "Prometheus",
        "query": "label_values(up{job=\"node\"}, instance)",
        "refresh": 2,
        "multi": true,
        "includeAll": true
      },
      {
        "name": "interval",
        "type": "interval",
        "options": [
          {"text": "1m", "value": "1m"},
          {"text": "5m", "value": "5m"},
          {"text": "1h", "value": "1h"}
        ],
        "auto": true,
        "auto_min": "1m"
      }
    ]
  }
}

# 使用單選變數
node_cpu_seconds_total{instance="$instance"}

# 使用多選變數
node_cpu_seconds_total{instance=~"$instance"}

# 使用 interval 變數
rate(http_requests_total[$interval])

# /etc/grafana/provisioning/dashboards/default.yml
apiVersion: 1

providers:
  - name: 'Infrastructure'
    orgId: 1
    folder: 'Infrastructure'
    type: file
    disableDeletion: true
    editable: false
    options:
      path: /var/lib/grafana/dashboards/infrastructure

  - name: 'Applications'
    orgId: 1
    folder: 'Applications'
    type: file
    disableDeletion: true
    editable: false
    options:
      path: /var/lib/grafana/dashboards/applications

# 即時查詢 - Instant Vector
http_requests_total

# 帶 Label 過濾
http_requests_total{job="api", status="200"}

# 範圍查詢 - Range Vector
http_requests_total[5m]

# Label 匹配運算子
http_requests_total{status="200"}        # 完全匹配
http_requests_total{status!="500"}       # 不等於
http_requests_total{status=~"2.."}       # 正則匹配
http_requests_total{status!~"5.."}       # 正則不匹配

# rate() - 計算每秒增長率（Counter 專用）
rate(http_requests_total[5m])

# irate() - 瞬時增長率（更敏感）
irate(http_requests_total[5m])

# increase() - 區間內增量
increase(http_requests_total[1h])

# sum() - 加總
sum(rate(http_requests_total[5m])) by (job)

# avg() - 平均
avg(node_cpu_seconds_total{mode="idle"}) by (instance)

# max() / min() - 最大/最小值
max(node_memory_MemAvailable_bytes) by (instance)

# histogram_quantile() - 百分位數
histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m]))

# 計算 CPU 使用率
100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# 計算記憶體使用率
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100

# 計算磁碟使用率
(1 - node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} 
    / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) * 100

# HTTP 錯誤率
sum(rate(http_requests_total{status=~"5.."}[5m])) 
  / sum(rate(http_requests_total[5m])) * 100

# P99 Latency
histogram_quantile(0.99, 
  sum(rate(http_request_duration_seconds_bucket[5m])) by (le, endpoint))

# 預測磁碟空間（線性回歸）
predict_linear(node_filesystem_avail_bytes[1h], 24*3600)

# CPU 使用率（按模式）
sum by (mode) (rate(node_cpu_seconds_total[5m]))

# 系統負載
node_load1   # 1 分鐘平均
node_load5   # 5 分鐘平均
node_load15  # 15 分鐘平均

# CPU 飽和度（Load > CPU 數量）
node_load1 / count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})

# 記憶體可用量
node_memory_MemAvailable_bytes

# 記憶體使用量
node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes

# Swap 使用量
node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes

# 磁碟可用空間
node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"}

# 磁碟 I/O 速率
rate(node_disk_read_bytes_total[5m])
rate(node_disk_written_bytes_total[5m])

# 磁碟 IOPS
rate(node_disk_reads_completed_total[5m])
rate(node_disk_writes_completed_total[5m])

# Heap 使用量
jvm_memory_used_bytes{area="heap"}
jvm_memory_max_bytes{area="heap"}

# GC 次數與時間
rate(jvm_gc_pause_seconds_count[5m])
rate(jvm_gc_pause_seconds_sum[5m])

# 執行緒數
jvm_threads_live_threads
jvm_threads_peak_threads

# 請求速率 (QPS)
sum(rate(http_server_requests_seconds_count[5m])) by (uri)

# 平均延遲
rate(http_server_requests_seconds_sum[5m]) 
  / rate(http_server_requests_seconds_count[5m])

# 錯誤率
sum(rate(http_server_requests_seconds_count{status=~"5.."}[5m])) 
  / sum(rate(http_server_requests_seconds_count[5m])) * 100

{
  "panels": [
    {
      "title": "P99 Latency",
      "type": "stat",
      "targets": [
        {
          "expr": "histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job=\"api\"}[5m])) by (le))",
          "legendFormat": "P99"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "s",
          "thresholds": {
            "steps": [
              {"color": "green", "value": null},
              {"color": "yellow", "value": 0.5},
              {"color": "red", "value": 1}
            ]
          }
        }
      }
    },
    {
      "title": "Latency Distribution",
      "type": "heatmap",
      "targets": [
        {
          "expr": "sum(rate(http_request_duration_seconds_bucket{job=\"api\"}[5m])) by (le)",
          "format": "heatmap"
        }
      ]
    }
  ]
}

# Batch 執行成功率
sum(batch_job_completed_total{status="success"}) 
  / sum(batch_job_completed_total) * 100

# Batch 執行時間
batch_job_duration_seconds{quantile="0.99"}

# 失敗的 Batch Jobs
batch_job_completed_total{status="failed"}

你是 Prometheus 專家。請幫我寫 PromQL 查詢：

需求：
- 計算過去 5 分鐘內，所有 API endpoint 的 P95 延遲
- 依據 endpoint 分組
- 只顯示延遲超過 500ms 的 endpoint

可用的 Metrics：
- http_request_duration_seconds_bucket (Histogram)
- Labels: job, endpoint, method, status

請幫我設計一個「JVM 監控 Dashboard」的 JSON 結構：

需求：
1. 第一行：4 個 Stat Panel（Heap 使用率、GC 次數、Thread 數、Uptime）
2. 第二行：Heap 使用趨勢圖、GC 暫停時間趨勢圖
3. 第三行：Thread 狀態分布、Class 載入數量

請使用 Grafana Dashboard JSON 格式輸出。

# alertmanager.yml
global:
  smtp_smarthost: 'smtp.example.com:587'
  smtp_from: 'alertmanager@example.com'
  smtp_auth_username: 'alertmanager'
  smtp_auth_password: 'password'

route:
  receiver: 'default'
  group_by: ['alertname', 'env', 'severity']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  routes:
    - match:
        severity: critical
      receiver: 'pagerduty-critical'
      continue: true
    - match:
        severity: critical
      receiver: 'slack-critical'
    - match:
        severity: warning
      receiver: 'slack-warning'

receivers:
  - name: 'default'
    email_configs:
      - to: 'ops-team@example.com'

  - name: 'pagerduty-critical'
    pagerduty_configs:
      - service_key: '<your-service-key>'
        severity: critical

  - name: 'slack-critical'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/xxx'
        channel: '#alerts-critical'
        title: '🚨 Critical Alert'
        text: '{{ .CommonAnnotations.description }}'

  - name: 'slack-warning'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/xxx'
        channel: '#alerts-warning'
        title: '⚠️ Warning Alert'

inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'instance']

# /etc/prometheus/rules/node_alerts.yml
groups:
  - name: node_alerts
    rules:
      # CPU 高使用率告警
      - alert: HighCpuUsage
        expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High CPU usage on {{ $labels.instance }}"
          description: "CPU usage is {{ $value | printf \"%.2f\" }}% on {{ $labels.instance }}"

      # CPU 極高使用率告警
      - alert: CriticalCpuUsage
        expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Critical CPU usage on {{ $labels.instance }}"
          description: "CPU usage is {{ $value | printf \"%.2f\" }}% on {{ $labels.instance }}"

      # 記憶體不足告警
      - alert: LowMemory
        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Low memory on {{ $labels.instance }}"
          description: "Memory usage is {{ $value | printf \"%.2f\" }}%"

      # 磁碟空間不足告警
      - alert: LowDiskSpace
        expr: (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes) * 100 > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Low disk space on {{ $labels.instance }}"
          description: "Disk usage is {{ $value | printf \"%.2f\" }}% on {{ $labels.mountpoint }}"

      # 服務 Down 告警
      - alert: InstanceDown
        expr: up == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Instance {{ $labels.instance }} is down"
          description: "{{ $labels.job }} on {{ $labels.instance }} has been down for more than 1 minute"

# /etc/prometheus/rules/app_alerts.yml
groups:
  - name: application_alerts
    rules:
      # API 錯誤率告警
      - alert: HighErrorRate
        expr: |
          sum(rate(http_server_requests_seconds_count{status=~"5.."}[5m])) by (application)
          / sum(rate(http_server_requests_seconds_count[5m])) by (application) * 100 > 1
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "High error rate on {{ $labels.application }}"
          description: "Error rate is {{ $value | printf \"%.2f\" }}%"

      # API 延遲告警
      - alert: HighLatency
        expr: |
          histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, application)) > 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High P99 latency on {{ $labels.application }}"
          description: "P99 latency is {{ $value | printf \"%.2f\" }}s"

      # JVM Heap 告警
      - alert: JvmHeapHigh
        expr: jvm_memory_used_bytes{area="heap"} / jvm_memory_max_bytes{area="heap"} * 100 > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High JVM heap usage on {{ $labels.application }}"
          description: "Heap usage is {{ $value | printf \"%.2f\" }}%"

# alertmanager.yml
receivers:
  - name: 'teams-alerts'
    webhook_configs:
      - url: 'https://outlook.office.com/webhook/xxx'
        send_resolved: true

receivers:
  - name: 'slack-alerts'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/T00/B00/xxx'
        channel: '#alerts'
        username: 'Prometheus'
        icon_emoji: ':prometheus:'
        title: '{{ .Status | toUpper }}{{ if eq .Status "firing" }} 🔥{{ else }} ✅{{ end }}'
        text: >-
          {{ range .Alerts }}
          *Alert:* {{ .Annotations.summary }}
          *Description:* {{ .Annotations.description }}
          *Severity:* {{ .Labels.severity }}
          {{ end }}

儲存空間 = 時序數量 × 每秒樣本數 × 每樣本大小 × 保留天數

範例：
- 10,000 個時序
- 每 15 秒一個樣本
- 每樣本約 2 bytes
- 保留 15 天

= 10,000 × (86400/15) × 2 × 15
= 10,000 × 5,760 × 2 × 15
≈ 1.7 GB（壓縮後約 0.5 GB）

# TSDB 目前大小
prometheus_tsdb_storage_blocks_bytes

# 時序數量
prometheus_tsdb_head_series

# 樣本數量
prometheus_tsdb_head_samples_appended_total

# WAL 大小
prometheus_tsdb_wal_storage_size_bytes

# 方法一：調整保留時間
--storage.tsdb.retention.time=7d

# 方法二：設定大小上限
--storage.tsdb.retention.size=30GB

# 方法三：手動觸發壓縮（謹慎使用）
curl -X POST http://localhost:9090/api/v1/admin/tsdb/clean_tombstones

# 方法四：刪除過期資料（需啟用 admin API）
curl -X POST http://localhost:9090/api/v1/admin/tsdb/delete_series \
  -d 'match[]=http_requests_total{job="old-service"}'

# 增加抓取並行度
global:
  scrape_interval: 15s
  scrape_timeout: 10s

# 在 scrape_config 中限制樣本數
scrape_configs:
  - job_name: 'high-cardinality-job'
    sample_limit: 10000  # 每次抓取最多樣本數

# 命令列調校參數
--storage.tsdb.min-block-duration=2h
--storage.tsdb.max-block-duration=36h
--query.max-samples=50000000
--query.timeout=2m

# /etc/grafana/grafana.ini
[server]
concurrent_render_request_limit = 30

[database]
max_idle_conn = 25
max_open_conn = 100
conn_max_lifetime = 14400

[dashboards]
min_refresh_interval = 15s

[dataproxy]
timeout = 60

# /etc/prometheus/rules/recording_rules.yml
groups:
  - name: recording_rules
    interval: 1m
    rules:
      # 預先計算 CPU 使用率
      - record: instance:node_cpu_utilization:avg5m
        expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

      # 預先計算 HTTP 錯誤率
      - record: job:http_error_rate:5m
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m])) by (job)
          / sum(rate(http_requests_total[5m])) by (job) * 100

      # 預先計算 P99 延遲
      - record: job:http_latency_p99:5m
        expr: histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, job))

# 方法一：快照備份（推薦）
curl -X POST http://localhost:9090/api/v1/admin/tsdb/snapshot
# 快照會存放在 /var/lib/prometheus/snapshots/

# 方法二：停機備份
sudo systemctl stop prometheus
tar -czvf prometheus-backup-$(date +%Y%m%d).tar.gz /var/lib/prometheus/
sudo systemctl start prometheus

# 方法三：遠端儲存
# 設定 remote_write 到 Thanos / Cortex

# 從快照還原
sudo systemctl stop prometheus
rm -rf /var/lib/prometheus/*
cp -r /var/lib/prometheus/snapshots/<snapshot-id>/* /var/lib/prometheus/
sudo chown -R prometheus:prometheus /var/lib/prometheus
sudo systemctl start prometheus

# 備份資料庫
cp /var/lib/grafana/grafana.db /backup/grafana-$(date +%Y%m%d).db

# 備份 provisioning 設定
tar -czvf grafana-provisioning-$(date +%Y%m%d).tar.gz /etc/grafana/provisioning/

# 使用 API 匯出 Dashboard
curl -H "Authorization: Bearer <api-key>" \
  http://localhost:3000/api/dashboards/uid/<uid> > dashboard-backup.json

#!/bin/bash
# backup-monitoring.sh

BACKUP_DIR="/backup/monitoring"
DATE=$(date +%Y%m%d)

# Prometheus 快照
curl -X POST http://localhost:9090/api/v1/admin/tsdb/snapshot

# Grafana 資料庫
cp /var/lib/grafana/grafana.db $BACKUP_DIR/grafana-$DATE.db

# 清理 7 天前的備份
find $BACKUP_DIR -type f -mtime +7 -delete

echo "Backup completed: $DATE"

# 1. 檢查版本相容性
# 參考：https://prometheus.io/docs/prometheus/latest/migration/

# 2. 備份資料
curl -X POST http://localhost:9090/api/v1/admin/tsdb/snapshot

# 3. 備份設定
cp -r /etc/prometheus /etc/prometheus.bak

# 4. 驗證設定檔
promtool check config /etc/prometheus/prometheus.yml
promtool check rules /etc/prometheus/rules/*.yml

# 1. 下載新版本
wget https://github.com/prometheus/prometheus/releases/download/v2.50.0/prometheus-2.50.0.linux-amd64.tar.gz

# 2. 解壓縮
tar xvfz prometheus-2.50.0.linux-amd64.tar.gz

# 3. 停止服務
sudo systemctl stop prometheus

# 4. 備份舊版本
sudo mv /usr/local/bin/prometheus /usr/local/bin/prometheus.old
sudo mv /usr/local/bin/promtool /usr/local/bin/promtool.old

# 5. 安裝新版本
sudo cp prometheus-2.50.0.linux-amd64/prometheus /usr/local/bin/
sudo cp prometheus-2.50.0.linux-amd64/promtool /usr/local/bin/

# 6. 啟動服務
sudo systemctl start prometheus

# 7. 驗證
prometheus --version
curl http://localhost:9090/-/healthy

# 1. 備份資料庫
cp /var/lib/grafana/grafana.db /backup/

# 2. 備份設定
cp /etc/grafana/grafana.ini /backup/

# 3. 匯出重要 Dashboard
# 使用 Web UI 或 API 匯出

# CentOS/RHEL
sudo yum update grafana

# Ubuntu/Debian
sudo apt-get update
sudo apt-get install --only-upgrade grafana

# 重啟服務
sudo systemctl restart grafana-server

# 1. 拉取新版本
docker pull grafana/grafana:10.3.0

# 2. 停止舊容器
docker stop grafana

# 3. 移除舊容器（資料卷保留）
docker rm grafana

# 4. 啟動新容器
docker run -d \
  --name grafana \
  --restart unless-stopped \
  -p 3000:3000 \
  -v /data/grafana:/var/lib/grafana \
  grafana/grafana:10.3.0

# Prometheus 回滾
sudo systemctl stop prometheus
sudo mv /usr/local/bin/prometheus /usr/local/bin/prometheus.new
sudo mv /usr/local/bin/prometheus.old /usr/local/bin/prometheus
sudo systemctl start prometheus

# Grafana 回滾（使用備份資料庫）
sudo systemctl stop grafana-server
cp /backup/grafana.db /var/lib/grafana/grafana.db
sudo yum downgrade grafana-<previous-version>  # 或使用 apt
sudo systemctl start grafana-server

<namespace>_<subsystem>_<name>_<unit>_<suffix>

範例：
http_server_requests_seconds_total      # HTTP 請求總數
http_server_requests_seconds_bucket     # HTTP 請求延遲分布
jvm_memory_used_bytes                   # JVM 記憶體使用量
node_cpu_seconds_total                  # CPU 使用時間

# ❌ 錯誤示範
request_count          # 缺少單位
httpRequestDuration    # 使用 camelCase
my.app.requests        # 使用點號
requests_2024          # 包含年份

# ✅ 好的設計
labels:
  env: production          # 環境
  region: taiwan           # 區域
  team: backend            # 團隊
  application: order-svc   # 應用程式
  instance: order-1        # 實例
  method: GET              # HTTP 方法
  status: 200              # HTTP 狀態碼
  endpoint: /api/orders    # API 端點

# ❌ 避免的設計
labels:
  user_id: 12345           # 用戶 ID（數百萬個值）
  request_id: abc-123      # 請求 ID（無限多個值）
  timestamp: 2024-01-01    # 時間戳
  email: user@example.com  # 個資
  trace_id: xxxx           # 追蹤 ID

# 查詢高 Cardinality 指標
topk(10, count by (__name__)({__name__=~".+"}))

# 查詢特定指標的 Label 數量
count(count by (user_id) (http_requests_total))

# prometheus-prod.yml
global:
  external_labels:
    env: production
    datacenter: primary

# prometheus-nonprod.yml
global:
  external_labels:
    env: non-production

scrape_configs:
  - job_name: 'dev-apps'
    static_configs:
      - targets: ['dev-app:8080']
        labels:
          env: dev

  - job_name: 'sit-apps'
    static_configs:
      - targets: ['sit-app:8080']
        labels:
          env: sit

# GitLab CI 範例
deploy:
  stage: deploy
  script:
    - kubectl apply -f k8s/
  after_script:
    # 驗證 Prometheus 已抓取新服務
    - |
      for i in {1..30}; do
        if curl -s "http://prometheus:9090/api/v1/targets" | grep -q "new-service"; then
          echo "Service registered in Prometheus"
          exit 0
        fi
        sleep 10
      done
      echo "Warning: Service not found in Prometheus"

// Spring Batch + Micrometer
@Component
public class BatchMetrics {
    private final MeterRegistry registry;
    
    public void recordJobExecution(String jobName, String status, Duration duration) {
        registry.counter("batch_job_executions_total", 
            "job", jobName, 
            "status", status).increment();
        
        registry.timer("batch_job_duration_seconds", 
            "job", jobName)
            .record(duration);
    }
}

# Batch 監控指標
batch_job_executions_total{status="COMPLETED"}
batch_job_executions_total{status="FAILED"}
batch_job_duration_seconds

# Istio Prometheus 整合
scrape_configs:
  - job_name: 'istio-mesh'
    kubernetes_sd_configs:
      - role: endpoints
        namespaces:
          names:
            - istio-system
    relabel_configs:
      - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
        action: keep
        regex: istio-telemetry;prometheus

# Prometheus 啟用基本認證
# 需搭配反向代理（如 Nginx）

# Grafana 安全設定
[security]
admin_password = <strong-password>
secret_key = <random-key>
disable_gravatar = true
cookie_secure = true
cookie_samesite = strict

[auth]
disable_login_form = false
oauth_auto_login = true

[auth.ldap]
enabled = true
config_file = /etc/grafana/ldap.toml

# Grafana Audit Log
[log]
mode = file
level = info

[auditing]
enabled = true
log_file = /var/log/grafana/audit.log

# 檢查時序數量
curl http://localhost:9090/api/v1/status/tsdb | jq '.data.headStats'

# 解決方案：
# 1. 減少 scrape 頻率
# 2. 移除不必要的 metrics
# 3. 降低 retention 時間
# 4. 使用 sample_limit 限制

# 調整 Grafana 逾時設定
[dataproxy]
timeout = 120
keep_alive_seconds = 60

# 優化查詢：使用 Recording Rules 預先計算

# 檢查 Alertmanager 狀態
curl http://localhost:9093/api/v1/status

# 檢查 Prometheus 告警狀態
curl http://localhost:9090/api/v1/alerts

# 常見原因：
# 1. 路由設定錯誤
# 2. 靜默（Silence）生效中
# 3. 通知管道設定錯誤

# 檢查網路連通性
curl -v http://target:9100/metrics

# 常見原因：
# 1. 防火牆阻擋
# 2. Exporter 未啟動
# 3. 錯誤的 Port
# 4. 認證失敗

# 找出高 Cardinality 指標
topk(10, count by (__name__)({__name__=~".+"}))

# 找出高 Cardinality Label
count by (job) (count by (job, instance) (up))

# 解決方案：
# 1. 移除不必要的 Label
# 2. 使用 relabel_configs 過濾
# 3. 限制 sample_limit