┌─────────────────────────────────────────────────────────────────┐
│                    OpenTelemetry 解決的問題                      │
├─────────────────────────────────────────────────────────────────┤
│  ❌ 傳統痛點                    │  ✅ OTel 解決方案              │
├─────────────────────────────────────────────────────────────────┤
│  • 各家 APM 工具 SDK 不相容      │  • 統一標準 API/SDK            │
│  • 廠商鎖定（Vendor Lock-in）   │  • 廠商中立（Vendor Neutral）  │
│  • Traces/Metrics/Logs 各自為政 │  • 三大支柱統一收集            │
│  • 切換監控工具成本高            │  • 後端可自由替換              │
└─────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────────┐
│  Trace 結構範例                                                  │
├─────────────────────────────────────────────────────────────────┤
│  Trace ID: abc123                                               │
│  ├── Span: API Gateway (50ms)                                   │
│  │   └── Span: User Service (30ms)                              │
│  │       └── Span: Database Query (15ms)                        │
│  └── Span: Order Service (40ms)                                 │
│      └── Span: Payment Service (25ms)                           │
└─────────────────────────────────────────────────────────────────┘

// Java API 範例：僅使用介面
import io.opentelemetry.api.trace.Tracer;
import io.opentelemetry.api.trace.Span;

Tracer tracer = GlobalOpenTelemetry.getTracer("my-service");
Span span = tracer.spanBuilder("my-operation").startSpan();

// Java SDK 初始化範例
SdkTracerProvider tracerProvider = SdkTracerProvider.builder()
    .addSpanProcessor(BatchSpanProcessor.builder(otlpExporter).build())
    .setResource(Resource.create(Attributes.of(
        ResourceAttributes.SERVICE_NAME, "my-service"
    )))
    .build();

┌─────────────────────────────────────────────────────────────────┐
│                    OTel Collector 架構                          │
├─────────────────────────────────────────────────────────────────┤
│                                                                 │
│   ┌──────────┐    ┌──────────┐    ┌──────────┐                 │
│   │ Receivers│ -> │Processors│ -> │ Exporters│                 │
│   └──────────┘    └──────────┘    └──────────┘                 │
│                                                                 │
│   接收資料          處理/轉換         匯出至後端                  │
│   • OTLP           • Batch          • Prometheus               │
│   • Jaeger         • Memory Limit   • Jaeger                   │
│   • Prometheus     • Attributes     • OTLP                     │
│   • Zipkin         • Filter         • Elasticsearch            │
│                                                                 │
└─────────────────────────────────────────────────────────────────┘

# Linux amd64
curl -LO https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.96.0/otelcol-contrib_0.96.0_linux_amd64.tar.gz

# 解壓縮
tar -xzf otelcol-contrib_0.96.0_linux_amd64.tar.gz

# 驗證安裝
./otelcol-contrib --version

# 建立設定目錄
mkdir -p /etc/otelcol

# 建立基本設定檔
cat > /etc/otelcol/config.yaml << 'EOF'
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:
    timeout: 1s
    send_batch_size: 1024

exporters:
  debug:
    verbosity: detailed

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [debug]
    metrics:
      receivers: [otlp]
      processors: [batch]
      exporters: [debug]
    logs:
      receivers: [otlp]
      processors: [batch]
      exporters: [debug]
EOF

./otelcol-contrib --config=/etc/otelcol/config.yaml

# 建立 systemd service 檔案
cat > /etc/systemd/system/otelcol.service << 'EOF'
[Unit]
Description=OpenTelemetry Collector
After=network.target

[Service]
Type=simple
User=otel
Group=otel
ExecStart=/usr/local/bin/otelcol-contrib --config=/etc/otelcol/config.yaml
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

# 啟用並啟動服務
systemctl daemon-reload
systemctl enable otelcol
systemctl start otelcol

# 檢查狀態
systemctl status otelcol

docker run -d \
  --name otel-collector \
  -p 4317:4317 \
  -p 4318:4318 \
  -p 8888:8888 \
  -v $(pwd)/config.yaml:/etc/otelcol-contrib/config.yaml \
  otel/opentelemetry-collector-contrib:0.96.0

# docker-compose.yaml
version: '3.8'

services:
  otel-collector:
    image: otel/opentelemetry-collector-contrib:0.96.0
    container_name: otel-collector
    command: ["--config=/etc/otelcol-contrib/config.yaml"]
    volumes:
      - ./config.yaml:/etc/otelcol-contrib/config.yaml:ro
    ports:
      - "4317:4317"   # OTLP gRPC
      - "4318:4318"   # OTLP HTTP
      - "8888:8888"   # Prometheus metrics (self)
      - "8889:8889"   # Prometheus exporter
      - "13133:13133" # Health check
    restart: unless-stopped
    
  jaeger:
    image: jaegertracing/all-in-one:1.54
    container_name: jaeger
    ports:
      - "16686:16686" # Jaeger UI
      - "14250:14250" # gRPC
    restart: unless-stopped

  prometheus:
    image: prom/prometheus:v2.50.0
    container_name: prometheus
    volumes:
      - ./prometheus.yaml:/etc/prometheus/prometheus.yml:ro
    ports:
      - "9090:9090"
    restart: unless-stopped

  grafana:
    image: grafana/grafana:10.3.0
    container_name: grafana
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    restart: unless-stopped

helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
helm repo update

# otel-collector-values.yaml
mode: deployment  # 或 daemonset

config:
  receivers:
    otlp:
      protocols:
        grpc:
          endpoint: 0.0.0.0:4317
        http:
          endpoint: 0.0.0.0:4318
    
  processors:
    batch:
      timeout: 1s
      send_batch_size: 1024
    memory_limiter:
      check_interval: 1s
      limit_mib: 512
      spike_limit_mib: 128
    k8sattributes:
      extract:
        metadata:
          - k8s.namespace.name
          - k8s.deployment.name
          - k8s.pod.name
          - k8s.node.name
    
  exporters:
    otlp/jaeger:
      endpoint: jaeger-collector.observability:4317
      tls:
        insecure: true
    prometheusremotewrite:
      endpoint: http://prometheus.observability:9090/api/v1/write
    
  service:
    pipelines:
      traces:
        receivers: [otlp]
        processors: [memory_limiter, k8sattributes, batch]
        exporters: [otlp/jaeger]
      metrics:
        receivers: [otlp]
        processors: [memory_limiter, batch]
        exporters: [prometheusremotewrite]

resources:
  limits:
    cpu: 500m
    memory: 512Mi
  requests:
    cpu: 100m
    memory: 128Mi

serviceAccount:
  create: true

clusterRole:
  create: true
  rules:
    - apiGroups: [""]
      resources: ["pods", "namespaces"]
      verbs: ["get", "watch", "list"]

# 建立 namespace
kubectl create namespace observability

# 安裝 Collector
helm install otel-collector open-telemetry/opentelemetry-collector \
  -n observability \
  -f otel-collector-values.yaml

# 驗證安裝
kubectl get pods -n observability

processors:
  k8sattributes:
    auth_type: "serviceAccount"
    passthrough: false
    extract:
      metadata:
        - k8s.namespace.name
        - k8s.deployment.name
        - k8s.statefulset.name
        - k8s.daemonset.name
        - k8s.cronjob.name
        - k8s.job.name
        - k8s.pod.name
        - k8s.pod.uid
        - k8s.node.name
        - container.id
        - container.image.name
        - container.image.tag
      labels:
        - tag_name: app
          key: app.kubernetes.io/name
        - tag_name: version
          key: app.kubernetes.io/version
    pod_association:
      - sources:
          - from: resource_attribute
            name: k8s.pod.ip

# config.yaml 基本結構
receivers:     # 接收資料的來源
  ...

processors:    # 資料處理與轉換
  ...

exporters:     # 匯出資料的目的地
  ...

extensions:    # 擴充功能（健康檢查、認證等）
  ...

service:       # 定義資料處理管線
  extensions: [...]
  pipelines:
    traces:
      receivers: [...]
      processors: [...]
      exporters: [...]
    metrics:
      receivers: [...]
      processors: [...]
      exporters: [...]
    logs:
      receivers: [...]
      processors: [...]
      exporters: [...]

# /etc/otelcol/config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
        max_recv_msg_size_mib: 4
      http:
        endpoint: 0.0.0.0:4318
        cors:
          allowed_origins:
            - "http://*"
            - "https://*"

  # 收集主機指標
  hostmetrics:
    collection_interval: 30s
    scrapers:
      cpu:
      memory:
      disk:
      network:

processors:
  # 記憶體限制（必要）
  memory_limiter:
    check_interval: 1s
    limit_mib: 1024
    spike_limit_mib: 256

  # 批次處理
  batch:
    timeout: 5s
    send_batch_size: 2048
    send_batch_max_size: 4096

  # 資源屬性設定
  resource:
    attributes:
      - key: environment
        value: production
        action: upsert
      - key: service.namespace
        value: my-company
        action: upsert

  # 屬性處理
  attributes:
    actions:
      - key: db.password
        action: delete
      - key: http.request.header.authorization
        action: delete

  # 尾端採樣（Traces 專用）
  tail_sampling:
    decision_wait: 10s
    num_traces: 100000
    policies:
      - name: errors-policy
        type: status_code
        status_code:
          status_codes: [ERROR]
      - name: slow-traces-policy
        type: latency
        latency:
          threshold_ms: 1000
      - name: probabilistic-policy
        type: probabilistic
        probabilistic:
          sampling_percentage: 10

exporters:
  # 匯出至 Jaeger
  otlp/jaeger:
    endpoint: jaeger-collector:4317
    tls:
      insecure: true

  # 匯出至 Prometheus
  prometheusremotewrite:
    endpoint: http://prometheus:9090/api/v1/write
    tls:
      insecure: true
    resource_to_telemetry_conversion:
      enabled: true

  # 匯出至 Elasticsearch
  elasticsearch:
    endpoints: [http://elasticsearch:9200]
    logs_index: otel-logs
    traces_index: otel-traces

  # 除錯輸出
  debug:
    verbosity: basic

extensions:
  health_check:
    endpoint: 0.0.0.0:13133
  
  zpages:
    endpoint: 0.0.0.0:55679

  pprof:
    endpoint: 0.0.0.0:1777

service:
  extensions: [health_check, zpages, pprof]
  
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, attributes, tail_sampling, batch]
      exporters: [otlp/jaeger, elasticsearch]
    
    metrics:
      receivers: [otlp, hostmetrics]
      processors: [memory_limiter, resource, batch]
      exporters: [prometheusremotewrite]
    
    logs:
      receivers: [otlp]
      processors: [memory_limiter, attributes, batch]
      exporters: [elasticsearch]

  telemetry:
    logs:
      level: info
    metrics:
      address: 0.0.0.0:8888

processors:
  # 1. memory_limiter - 必須放在最前面
  memory_limiter:
    check_interval: 1s
    limit_mib: 1024
    spike_limit_mib: 256

  # 2. batch - 必須放在 exporter 之前
  batch:
    timeout: 5s
    send_batch_size: 2048

processors:
  attributes:
    actions:
      # 刪除敏感資料
      - key: db.password
        action: delete
      - key: http.request.header.authorization
        action: delete
      - key: user.password
        action: delete
      
      # 遮罩處理
      - key: user.email
        pattern: ^(.{3}).*(@.*)$
        replacement: $1***$2
        action: update

processors:
  # 生產環境採樣策略
  tail_sampling:
    decision_wait: 10s
    policies:
      # 所有錯誤都保留
      - name: errors
        type: status_code
        status_code:
          status_codes: [ERROR]
      # 慢請求都保留
      - name: slow
        type: latency
        latency:
          threshold_ms: 500
      # 其餘 10% 採樣
      - name: default
        type: probabilistic
        probabilistic:
          sampling_percentage: 10

# 下載 Agent
curl -LO https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.1.0/opentelemetry-javaagent.jar

# 啟動應用
java -javaagent:opentelemetry-javaagent.jar \
     -Dotel.service.name=my-service \
     -Dotel.exporter.otlp.endpoint=http://localhost:4317 \
     -jar my-application.jar

<dependencyManagement>
    <dependencies>
        <dependency>
            <groupId>io.opentelemetry</groupId>
            <artifactId>opentelemetry-bom</artifactId>
            <version>1.35.0</version>
            <type>pom</type>
            <scope>import</scope>
        </dependency>
    </dependencies>
</dependencyManagement>

<dependencies>
    <!-- OpenTelemetry API -->
    <dependency>
        <groupId>io.opentelemetry</groupId>
        <artifactId>opentelemetry-api</artifactId>
    </dependency>
    
    <!-- OpenTelemetry SDK -->
    <dependency>
        <groupId>io.opentelemetry</groupId>
        <artifactId>opentelemetry-sdk</artifactId>
    </dependency>
    
    <!-- OTLP Exporter -->
    <dependency>
        <groupId>io.opentelemetry</groupId>
        <artifactId>opentelemetry-exporter-otlp</artifactId>
    </dependency>
    
    <!-- Spring Boot Starter（可選） -->
    <dependency>
        <groupId>io.opentelemetry.instrumentation</groupId>
        <artifactId>opentelemetry-spring-boot-starter</artifactId>
        <version>2.1.0-alpha</version>
    </dependency>
</dependencies>

# application.properties 或環境變數
OTEL_SERVICE_NAME=order-service
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
OTEL_EXPORTER_OTLP_PROTOCOL=grpc
OTEL_TRACES_SAMPLER=parentbased_traceidratio
OTEL_TRACES_SAMPLER_ARG=0.1
OTEL_RESOURCE_ATTRIBUTES=service.namespace=ecommerce,deployment.environment=production

import io.opentelemetry.api.GlobalOpenTelemetry;
import io.opentelemetry.api.trace.Span;
import io.opentelemetry.api.trace.SpanKind;
import io.opentelemetry.api.trace.StatusCode;
import io.opentelemetry.api.trace.Tracer;
import io.opentelemetry.context.Scope;

@Service
public class OrderService {
    
    private static final Tracer tracer = GlobalOpenTelemetry.getTracer("order-service");
    
    public Order createOrder(OrderRequest request) {
        // 建立 Span
        Span span = tracer.spanBuilder("createOrder")
                .setSpanKind(SpanKind.INTERNAL)
                .setAttribute("order.type", request.getType())
                .setAttribute("customer.id", request.getCustomerId())
                .startSpan();
        
        try (Scope scope = span.makeCurrent()) {
            // 業務邏輯
            validateOrder(request);
            Order order = processOrder(request);
            
            // 記錄成功資訊
            span.setAttribute("order.id", order.getId());
            span.setStatus(StatusCode.OK);
            
            return order;
            
        } catch (Exception e) {
            // 記錄錯誤
            span.recordException(e);
            span.setStatus(StatusCode.ERROR, e.getMessage());
            throw e;
            
        } finally {
            span.end();
        }
    }
    
    private void validateOrder(OrderRequest request) {
        Span span = tracer.spanBuilder("validateOrder")
                .startSpan();
        try (Scope scope = span.makeCurrent()) {
            // 驗證邏輯
        } finally {
            span.end();
        }
    }
}

# application.yaml
spring:
  application:
    name: order-service

otel:
  exporter:
    otlp:
      endpoint: http://otel-collector:4317
  resource:
    attributes:
      service.namespace: ecommerce
      deployment.environment: ${ENVIRONMENT:development}
  traces:
    sampler:
      probability: 0.1

@RestController
@RequestMapping("/api/orders")
public class OrderController {
    
    private final OrderService orderService;
    private final Tracer tracer;
    
    public OrderController(OrderService orderService, Tracer tracer) {
        this.orderService = orderService;
        this.tracer = tracer;
    }
    
    @PostMapping
    public ResponseEntity<Order> createOrder(@RequestBody OrderRequest request) {
        // Spring Web 會自動建立 Span
        // 可在此加入額外屬性
        Span currentSpan = Span.current();
        currentSpan.setAttribute("http.request.body.size", request.toString().length());
        
        Order order = orderService.createOrder(request);
        return ResponseEntity.ok(order);
    }
}

npm install @opentelemetry/api \
            @opentelemetry/sdk-node \
            @opentelemetry/auto-instrumentations-node \
            @opentelemetry/exporter-trace-otlp-grpc \
            @opentelemetry/exporter-metrics-otlp-grpc

// tracing.js - 必須在應用啟動前載入
const { NodeSDK } = require('@opentelemetry/sdk-node');
const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node');
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-grpc');
const { OTLPMetricExporter } = require('@opentelemetry/exporter-metrics-otlp-grpc');
const { PeriodicExportingMetricReader } = require('@opentelemetry/sdk-metrics');
const { Resource } = require('@opentelemetry/resources');
const { SemanticResourceAttributes } = require('@opentelemetry/semantic-conventions');

const resource = new Resource({
  [SemanticResourceAttributes.SERVICE_NAME]: 'user-service',
  [SemanticResourceAttributes.SERVICE_VERSION]: '1.0.0',
  [SemanticResourceAttributes.DEPLOYMENT_ENVIRONMENT]: process.env.NODE_ENV || 'development',
});

const traceExporter = new OTLPTraceExporter({
  url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4317',
});

const metricExporter = new OTLPMetricExporter({
  url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4317',
});

const sdk = new NodeSDK({
  resource,
  traceExporter,
  metricReader: new PeriodicExportingMetricReader({
    exporter: metricExporter,
    exportIntervalMillis: 60000,
  }),
  instrumentations: [getNodeAutoInstrumentations()],
});

sdk.start();

process.on('SIGTERM', () => {
  sdk.shutdown()
    .then(() => console.log('OTel SDK shut down'))
    .catch((error) => console.error('Error shutting down SDK', error))
    .finally(() => process.exit(0));
});

// app.js
require('./tracing'); // 必須在最前面

const express = require('express');
const { trace } = require('@opentelemetry/api');

const app = express();
const tracer = trace.getTracer('user-service');

app.get('/api/users/:id', async (req, res) => {
  const span = tracer.startSpan('getUserById');
  span.setAttribute('user.id', req.params.id);
  
  try {
    const user = await getUserFromDatabase(req.params.id);
    span.setStatus({ code: 1 }); // OK
    res.json(user);
  } catch (error) {
    span.recordException(error);
    span.setStatus({ code: 2, message: error.message }); // ERROR
    res.status(500).json({ error: error.message });
  } finally {
    span.end();
  }
});

app.listen(3000, () => {
  console.log('Server running on port 3000');
});

# 標準 Resource Attributes
service.name: order-service          # 服務名稱（必要）
service.version: 1.2.3               # 服務版本
service.namespace: ecommerce         # 服務命名空間
deployment.environment: production   # 部署環境
host.name: server-01                 # 主機名稱
container.id: abc123                 # 容器 ID
k8s.pod.name: order-service-xyz      # K8s Pod 名稱
k8s.namespace.name: production       # K8s Namespace

# HTTP 相關
http.method: POST
http.url: https://api.example.com/orders
http.status_code: 200
http.request.body.size: 1024

# 資料庫相關
db.system: postgresql
db.name: orders
db.operation: SELECT
db.statement: SELECT * FROM orders WHERE id = ?

# 自訂業務屬性
order.id: ORD-12345
customer.tier: premium

traceparent: 00-{trace-id}-{parent-id}-{trace-flags}
             |  |          |          |
             |  |          |          └── 01 = sampled
             |  |          └── 16 hex chars (8 bytes)
             |  └── 32 hex chars (16 bytes)
             └── version

範例：
traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01

@Configuration
public class RestTemplateConfig {
    
    @Bean
    public RestTemplate restTemplate() {
        // 使用 OTel Agent 時會自動注入
        return new RestTemplate();
    }
}

@Service
public class PaymentClient {
    
    private final RestTemplate restTemplate;
    
    public PaymentResult processPayment(PaymentRequest request) {
        // TraceContext 會自動透過 Header 傳遞
        return restTemplate.postForObject(
            "http://payment-service/api/payments",
            request,
            PaymentResult.class
        );
    }
}

import io.opentelemetry.context.Context;
import io.opentelemetry.context.propagation.TextMapSetter;

public class ContextPropagation {
    
    private static final TextMapSetter<HttpHeaders> setter = 
        (carrier, key, value) -> carrier.set(key, value);
    
    public void propagateContext(HttpHeaders headers) {
        GlobalOpenTelemetry.getPropagators()
            .getTextMapPropagator()
            .inject(Context.current(), headers, setter);
    }
}

Service: order-service
Operation: POST /api/orders
Min Duration: 500ms

┌─────────────────────────────────────────────────────────────────┐
│ Trace: POST /api/orders (總耗時: 850ms)                         │
├─────────────────────────────────────────────────────────────────┤
│ ├── order-service: createOrder (850ms)                         │
│ │   ├── order-service: validateOrder (50ms)                    │
│ │   ├── order-service: checkInventory (120ms)                  │
│ │   │   └── inventory-service: GET /api/inventory (110ms)      │
│ │   ├── order-service: processPayment (580ms) ⚠️ 瓶頸          │
│ │   │   └── payment-service: POST /api/payments (570ms)        │
│ │   │       └── payment-service: callExternalGateway (550ms)   │
│ │   └── order-service: saveOrder (80ms)                        │
│ │       └── database: INSERT orders (75ms)                     │
└─────────────────────────────────────────────────────────────────┘

搜尋條件：
Service: order-service
Tags: error=true
        otel.status_code=ERROR

{
  "traceId": "abc123...",
  "spanId": "def456...",
  "operationName": "processPayment",
  "status": {
    "code": "ERROR",
    "message": "Connection timeout"
  },
  "events": [
    {
      "name": "exception",
      "attributes": {
        "exception.type": "java.net.SocketTimeoutException",
        "exception.message": "Connect timed out",
        "exception.stacktrace": "..."
      }
    }
  ]
}

# grafana/provisioning/datasources/datasources.yaml
apiVersion: 1

datasources:
  - name: Jaeger
    type: jaeger
    access: proxy
    url: http://jaeger-query:16686
    isDefault: false
    
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true

{
  "links": [
    {
      "title": "View Traces",
      "url": "/explore?orgId=1&left={\"datasource\":\"Jaeger\",\"queries\":[{\"refId\":\"A\",\"service\":\"${service}\",\"tags\":\"http.status_code=${status_code}\"}]}",
      "targetBlank": true
    }
  ]
}

histogram_quantile(0.99, 
  sum(rate(http_server_duration_milliseconds_bucket{service_name="order-service"}[5m])) 
  by (le, http_route)
)

sum(rate(http_server_duration_milliseconds_count{service_name="order-service", http_status_code=~"5.."}[5m]))
/
sum(rate(http_server_duration_milliseconds_count{service_name="order-service"}[5m]))

發現：payment-service 回傳 200，但內容為 {"status": "PENDING"}
根本原因：第三方支付閘道處理中，但 order-service 誤判為成功
解決方案：修正狀態判斷邏輯，加入 PENDING 狀態處理

發現：database: SELECT 從 20ms 上升到 400ms
根本原因：資料表資料量增加，缺少索引
解決方案：新增適當的資料庫索引

# otel-collector-ha-values.yaml
mode: deployment

replicaCount: 3

podDisruptionBudget:
  enabled: true
  minAvailable: 2

resources:
  limits:
    cpu: 1000m
    memory: 2Gi
  requests:
    cpu: 200m
    memory: 400Mi

autoscaling:
  enabled: true
  minReplicas: 3
  maxReplicas: 10
  targetCPUUtilizationPercentage: 70
  targetMemoryUtilizationPercentage: 80

affinity:
  podAntiAffinity:
    preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 100
        podAffinityTerm:
          labelSelector:
            matchExpressions:
              - key: app.kubernetes.io/name
                operator: In
                values:
                  - opentelemetry-collector
          topologyKey: kubernetes.io/hostname

service:
  type: ClusterIP

processors:
  batch:
    # 增加批次大小提升吞吐量
    send_batch_size: 4096
    send_batch_max_size: 8192
    timeout: 5s

  memory_limiter:
    # 設定合理的記憶體限制
    check_interval: 1s
    limit_mib: 1800      # 實際限制
    spike_limit_mib: 400  # 突發限制

exporters:
  otlp:
    # 連線池設定
    sending_queue:
      enabled: true
      num_consumers: 10
      queue_size: 5000
    retry_on_failure:
      enabled: true
      initial_interval: 5s
      max_interval: 30s
      max_elapsed_time: 300s

extensions:
  zpages:
    endpoint: 0.0.0.0:55679

service:
  extensions: [zpages]

extensions:
  pprof:
    endpoint: 0.0.0.0:1777

service:
  extensions: [pprof]

# CPU profiling
go tool pprof http://collector:1777/debug/pprof/profile?seconds=30

# Memory profiling
go tool pprof http://collector:1777/debug/pprof/heap

service:
  telemetry:
    logs:
      level: info
      encoding: json
    metrics:
      address: 0.0.0.0:8888
      level: detailed

# prometheus-alerts.yaml
groups:
  - name: otel-collector-alerts
    rules:
      - alert: OTelCollectorExportFailure
        expr: rate(otelcol_exporter_send_failed_spans[5m]) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "OTel Collector export failure"
          description: "Collector {{ $labels.instance }} has export failures"

      - alert: OTelCollectorHighMemory
        expr: otelcol_process_memory_rss / 1024 / 1024 > 1500
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "OTel Collector high memory usage"
          description: "Collector {{ $labels.instance }} memory usage is above 1.5GB"

      - alert: OTelCollectorQueueFull
        expr: otelcol_exporter_queue_size / otelcol_exporter_queue_capacity > 0.8
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "OTel Collector queue near capacity"
          description: "Collector {{ $labels.instance }} queue is {{ $value | humanizePercentage }} full"

# 1. 更新 Helm values（指定新版本）
# otel-collector-values.yaml
# image:
#   tag: 0.96.0  # 新版本

# 2. 預覽變更
helm diff upgrade otel-collector open-telemetry/opentelemetry-collector \
  -n observability \
  -f otel-collector-values.yaml

# 3. 執行升級（滾動更新）
helm upgrade otel-collector open-telemetry/opentelemetry-collector \
  -n observability \
  -f otel-collector-values.yaml

# 4. 監控升級狀態
kubectl rollout status deployment/otel-collector -n observability

# 5. 如需回滾
kubectl rollout undo deployment/otel-collector -n observability

# 推薦格式
service.name: {team}-{application}-{component}
service.namespace: {business-domain}

# 範例
service.name: payment-gateway-api
service.namespace: financial-services

# 遵循 OpenTelemetry Semantic Conventions
# https://opentelemetry.io/docs/concepts/semantic-conventions/

# 自訂業務屬性使用 namespace
app.order.id: ORD-12345
app.customer.tier: premium
app.transaction.type: purchase

# 避免
order_id: ...       # 缺少 namespace
orderId: ...        # 不一致的命名風格

# 生產環境安全設定
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
        tls:
          cert_file: /certs/server.crt
          key_file: /certs/server.key
          client_ca_file: /certs/ca.crt
        auth:
          authenticator: oidc

extensions:
  oidc:
    issuer_url: https://auth.company.com
    audience: otel-collector

processors:
  # 敏感資料過濾
  attributes:
    actions:
      - key: db.password
        action: delete
      - key: http.request.header.authorization
        action: delete
      - key: user.ssn
        pattern: "^\d{3}-\d{2}-\d{4}$"
        action: hash

# 驗證 Collector 設定
otelcol-contrib validate --config=config.yaml

# 檢查 Collector 狀態
curl http://localhost:13133/

# 檢查 Collector Metrics
curl http://localhost:8888/metrics

# Kubernetes 檢查
kubectl get pods -n observability
kubectl logs -f deployment/otel-collector -n observability
kubectl describe pod otel-collector-xxx -n observability