OpenTelemetry 面试题

Application → Instrumentation → API → SDK → Exporter → Backend
                              ↓
                        OTel Collector
                              ↓
                     Jaeger/Prometheus/OTLP

Trace Context (trace_id/span_id)
    ↓
关联 Metrics: Exemplar
    ↓ 携带 trace_id → 关联到具体请求
关联 Logs: span_id → 日志上下文注入
    ↓
统一查询："查看该请求的 Trace、Metrics 和 Logs"

服务A → HTTP Header (traceparent) → 服务B
           │                             │
      trace_id: abc              trace_id: abc
      span_id: 123               parent_span_id: 123
                                  span_id: 456

Trace (跟踪)
  ├── Span (SpanContext: trace_id=abc, span_id=123)
  │     ├── Span (SpanContext: trace_id=abc, span_id=456, parent=123)
  │     │     └── Span (SpanContext: trace_id=abc, span_id=789, parent=456)
  │     └── Span (SpanContext: trace_id=abc, span_id=aaa, parent=123)
  └── Span (SpanContext: trace_id=abc, span_id=bbb)

Traceparent Header: 版本-trace_id-span_id-trace_flags
示例：00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01

Receivers → Processors → Exporters
     │          │            │
  OTLP/    Batch/Sam   OTLP/
  Jaeger/  ple/Attr    Prometheus/
  Prometheus  修改    Jaeger/
                        Splunk/
                       自定义后端

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:
    timeout: 1s
    send_batch_size: 1024
  memory_limiter:
    check_interval: 1s
    limit_mib: 512

exporters:
  otlp:
    endpoint: jaeger:4317
    tls:
      insecure: true
  prometheus:
    endpoint: 0.0.0.0:8889

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [otlp]
    metrics:
      receivers: [otlp]
      processors: [batch]
      exporters: [prometheus]

# DaemonSet + Sidecar 模式
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: otel-agent
spec:
  selector:
    matchLabels:
      app: otel-agent
  template:
    metadata:
      labels:
        app: otel-agent
    spec:
      containers:
        - name: otel-agent
          image: otel/opentelemetry-collector-contrib:latest
          args:
            - --config=/etc/otel/config.yaml
          ports:
            - containerPort: 4317  # OTLP gRPC
            - containerPort: 4318  # OTLP HTTP
          resources:
            limits:
              memory: 256Mi
              cpu: 500m

// Go SDK ParentBased 采样：保留完整父子链
exporter, _ := otlptracegrpc.New(ctx)
tp := sdktrace.NewTracerProvider(
    sdktrace.WithSampler(
        sdktrace.ParentBased(sdktrace.TraceIDRatioBased(0.1)),
    ),
    sdktrace.WithBatcher(exporter),
)

processors:
  tail_sampling:
    decision_wait: 30s
    num_traces: 100000
    expected_new_traces_per_sec: 1000
    policies:
      # 保留所有错误请求
      - name: errors-policy
        type: status_code
        status_code:
          status_codes:
            - ERROR
      # 保留高延迟请求
      - name: slow-policy
        type: latency
        latency:
          threshold_ms: 5000
      # 按概率采样
      - name: probabilistic-policy
        type: probabilistic
        probabilistic:
          sampling_percentage: 10

exporters:
  otlp:
    endpoint: jaeger:4317

SDK TracerProvider
    ↓
SpanProcessor Chain
    ├── BatchSpanProcessor
    │     ├── Buffer → Export → Exporter → Backend
    │     └── 定时批量发送
    └── SimpleSpanProcessor
          └── Span → 立即 Exporter → Backend

Span 生命周期：
  Start → [Processor.OnStart] → End → [Processor.OnEnd] → Queue → Export

bsp := sdktrace.NewBatchSpanProcessor(
    exporter,
    sdktrace.WithBatchTimeout(5*time.Second),  // 5s 批量一次
    sdktrace.WithMaxExportBatchSize(512),       // 每批最大 512 Span
    sdktrace.WithMaxQueueSize(2048),            // 最大队列 2048
    sdktrace.WithExportTimeout(30*time.Second), // 导出超时 30s
)

type MetricSpanProcessor struct {
    next sdktrace.SpanProcessor
}

func (p *MetricSpanProcessor) OnStart(parent context.Context, s sdktrace.ReadWriteSpan) {
    // Span 开始时处理
    p.next.OnStart(parent, s)
}

func (p *MetricSpanProcessor) OnEnd(s sdktrace.ReadOnlySpan) {
    // Span 结束时处理（转换为 Metrics）
    duration := s.EndTime().Sub(s.StartTime())
    requestDuration.WithLabelValues(
        s.Attributes()["http.method"],
        s.Attributes()["http.status_code"],
    ).Observe(duration.Seconds())

    p.next.OnEnd(s)
}

func (p *MetricSpanProcessor) Shutdown(ctx context.Context) error {
    return p.next.Shutdown(ctx)
}

服务A                         服务B
  │                            │
  │── HTTP Request ──────────→│
  │   Headers:                 │
  │   traceparent: 00-abc-123 │  Extractor
  │   tracestate: vendor=val   │    ↓
  │                            │  SpanContext
  │   创建 Span                 │    │
  │   trace_id: abc            │  新建 Span
  │   span_id: 123             │  trace_id: abc
  │                            │  parent: 123
  │                            │  span_id: 456
  │                            │
  │←────────── Response ─────│

// Go SDK Propagation
import (
    "go.opentelemetry.io/otel/propagation"
)

// 组合多个 Propagator
tp := propagation.NewCompositeTextMapPropagator(
    propagation.TraceContext{},   // W3C TraceContext
    propagation.Baggage{},        // W3C Baggage
)

// 注入到出口请求
func inject(ctx context.Context, req *http.Request) {
    tp.Inject(ctx, propagation.HeaderCarrier(req.Header))
}

// 从入口请求提取
func extract(ctx context.Context, req *http.Request) context.Context {
    return tp.Extract(ctx, propagation.HeaderCarrier(req.Header))
}

# collector 配置
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317

processors:
  batch:
    timeout: 1s
    send_batch_size: 1000

exporters:
  prometheus:
    endpoint: 0.0.0.0:8889
    namespace: otel
    const_labels:
      env: production
    resource_to_telemetry_conversion:
      enabled: true

service:
  pipelines:
    metrics:
      receivers: [otlp]
      processors: [batch]
      exporters: [prometheus]

import (
    "go.opentelemetry.io/otel/exporters/prometheus"
    "go.opentelemetry.io/otel/sdk/metric"
)

func initMetrics() {
    exporter, err := prometheus.New()
    provider := metric.NewMeterProvider(
        metric.WithReader(exporter),
    )
    otel.SetMeterProvider(provider)
}

# prometheus.yml
scrape_configs:
  - job_name: otel-collector
    scrape_interval: 15s
    static_configs:
      - targets:
          - otel-collector:8889

  - job_name: otel-sdk-app
    scrape_interval: 15s
    static_configs:
      - targets:
          - app:9464

// HTTP 请求
span.SetAttributes(
    attribute.String("http.method", "GET"),
    attribute.String("http.url", "/api/users"),
    attribute.Int("http.status_code", 200),
    attribute.Int("http.request_content_length", 1024),
)

// 数据库查询
span.SetAttributes(
    attribute.String("db.system", "postgresql"),
    attribute.String("db.name", "users"),
    attribute.String("db.statement", "SELECT * FROM users"),
)

// 消息队列
span.SetAttributes(
    attribute.String("messaging.system", "kafka"),
    attribute.String("messaging.destination", "orders"),
    attribute.String("messaging.operation", "process"),
)

import "go.opentelemetry.io/otel/sdk/resource"

res, _ := resource.New(ctx,
    resource.WithAttributes(
        semconv.ServiceNameKey.String("user-service"),
        semconv.DeploymentEnvironmentKey.String("production"),
    ),
    resource.WithTelemetrySDK(),
    resource.WithHost(),
    resource.WithContainer(),
    resource.WithProcess(),
)

应用入口
  │
Detected Framework (e.g., net/http, Gin, Express)
  │
Instrumentation 拦截关键方法
  │
API → SDK → Span 创建/结束/注入 Context
  │
自动注入 traceparent header
  │
导出到 Collector/Backend

// 导入自动埋点包
import (
    "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
)

// 自动包装 HTTP Handler
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
    // 自动创建 Span
    // 自动注入 Context Propagation
    handleRequest(w, r)
})

wrapped := otelhttp.NewHandler(handler, "api.router")
http.Handle("/api/", wrapped)

# JVM 参数方式
java -javaagent:opentelemetry-javaagent.jar \
     -Dotel.service.name=user-service \
     -Dotel.traces.exporter=otlp \
     -Dotel.metrics.exporter=otlp \
     -Dotel.exporter.otlp.endpoint=http://otel-collector:4317 \
     -jar app.jar

import (
    "go.opentelemetry.io/otel/metric"
    "go.opentelemetry.io/otel/attribute"
)

meter := meterProvider.Meter("api-server")

// Counter：单调递增
requestCounter, _ := meter.Int64Counter(
    "api.requests.total",
    metric.WithDescription("Total API requests"),
)

// Histogram：值分布
latencyHistogram, _ := meter.Float64Histogram(
    "api.request.duration",
    metric.WithDescription("Request latency"),
    metric.WithUnit("ms"),
)

// UpDownCounter：可增减
activeConnections, _ := meter.Int64UpDownCounter(
    "api.active_connections",
    metric.WithDescription("Active connections"),
)

// 使用
requestCounter.Add(ctx, 1, attribute.String("method", "GET"))
latencyHistogram.Record(ctx, 150.0, attribute.String("status", "200"))
activeConnections.Add(ctx, 1)
activeConnections.Add(ctx, -1)

服务A                    服务B                    服务C
  │                       │                       │
  │── baggage:          →│                       │
  │   user_id=123,       │── baggage:            →│
  │   session_id=abc     │   user_id=123,         │
  │                       │   session_id=abc       │
  │                       │                       │
  │                       │   读取 Baggage         │
  │                       │   → 注入 Span           │
  │                       │   Attributes           │
  │                       │   → 影响采样决策        │

import (
    "go.opentelemetry.io/otel/baggage"
    "go.opentelemetry.io/otel/attribute"
)

// 设置 Baggage
member1, _ := baggage.NewMember("user_id", "123")
member2, _ := baggage.NewMember("experiment", "blue-v2")
bag, _ := baggage.New(member1, member2)
ctx = baggage.ContextWithBaggage(ctx, bag)

// 注入到 Span Attributes
span := trace.SpanFromContext(ctx)
for _, m := range baggage.FromContext(ctx).Members() {
    span.SetAttributes(attribute.String("baggage."+m.Key(), m.Value()))
}

// 在接收端提取
bag := baggage.FromContext(ctx)
userID := bag.Member("user_id").Value()

baggage: user_id=123,experiment=blue-v2,priority=high

TraceContext: 跟踪谁调用了谁（调用拓扑）
Baggage:     调用链中传递的业务数据（用户/实验/优先级）

TraceContext 是基础设施
Baggage 是业务数据通道

processors:
  batch:
    # 批量超时时间
    timeout: 1s
    # 每批最大 Span/Metric/Log 数
    send_batch_size: 8192
    # 队列最大容量
    send_batch_max_size: 10000

  batch/compressed:
    timeout: 5s
    send_batch_size: 512
    # 压缩
    metadata_keys:
      - "compression"

processors:
  memory_limiter:
    check_interval: 1s
    # 硬限制：超过时拒绝数据
    limit_mib: 512
    # 软限制：超过时开始降级
    spike_limit_mib: 100

    # 或使用百分比
    # limit_percentage: 70
    # spike_limit_percentage: 15

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317

processors:
  memory_limiter:
    check_interval: 1s
    limit_mib: 512
    spike_limit_mib: 100
  batch:
    timeout: 1s
    send_batch_size: 8192

exporters:
  otlp:
    endpoint: jaeger:4317

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [otlp]

正常状态：
  Memory < limit_mib → 正常处理

降级状态：
  Memory > limit_mib - spike_limit_mib
  → 拒绝新数据（返回 ResourceExhausted）
  → 等待 check_interval 后重新检查

恢复：
  Memory 下降至安全水位
  → 恢复正常处理

```mermaid
graph TD
    A["日志应用程序代码"] --> B["现有日志框架<br/>log4j/zap/slog"]
    B --> C["OTel Logs Bridge API<br/>（从现有框架桥接到 OTel）"]
    C --> D["OTel Logs SDK<br/>（聚合、采样、关联、导出）"]
    D --> E["Exporter"]
    E --> F["OTLP"]
    F --> G["Collector"]
    G --> H["后端"]

**关键概念：**

| 概念 | 说明 |
|------|------|
| **LogRecord** | 日志记录的数据模型 |
| **LogRecordProcessor** | 日志处理链（类比 SpanProcessor） |
| **LoggerProvider** | Logger 工厂 |
| **Logger** | 日志记录器 |
| **Bridge API** | 桥接外部日志库 |

**Go 集成示例：**
```go
import (
    "go.opentelemetry.io/otel/log"
    "go.opentelemetry.io/otel/sdk/log"
    "go.opentelemetry.io/otel/exporters/otlp/otlplog"
)

// 创建 Logger Provider
exporter, _ := otlplog.New(ctx)
processor := log.NewBatchProcessor(exporter)
provider := log.NewLoggerProvider(
    log.WithProcessor(processor),
    log.WithResource(res),
)

// 关联 Trace Context
ctx := context.WithValue(context.Background(), "trace_id", span.SpanContext().TraceID().String())
logger := provider.Logger("app-logger")
logger.Emit(log.Record{
    Timestamp: time.Now(),
    Severity: log.SeverityError,
    Body: log.StringValue("Database connection failed"),
    TraceID: span.SpanContext().TraceID(),
    SpanID: span.SpanContext().SpanID(),
})

// 日志自动注入 trace_id/span_id
// 通过 Context Propagation 实现
func logWithTrace(logger log.Logger, ctx context.Context, msg string) {
    span := trace.SpanFromContext(ctx)
    sc := span.SpanContext()
    logger.Emit(log.Record{
        Body: log.StringValue(msg),
        TraceID: sc.TraceID(),
        SpanID: sc.SpanID(),
    })
}

// Go SDK OTLP Exporter
import (
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
)

// gRPC 方式
exporter, _ := otlptracegrpc.New(ctx,
    otlptracegrpc.WithEndpoint("otel-collector:4317"),
    otlptracegrpc.WithInsecure(),
    otlptracegrpc.WithTimeout(10*time.Second),
)

// HTTP 方式
exporter, _ := otlptracehttp.New(ctx,
    otlptracehttp.WithEndpoint("otel-collector:4318"),
    otlptracehttp.WithInsecure(),
    otlptracehttp.WithTimeout(10*time.Second),
    otlptracehttp.WithCompression(otlptracehttp.GzipCompression),
)

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
        max_recv_msg_size_mib: 10
      http:
        endpoint: 0.0.0.0:4318
        cors:
          allowed_origins:
            - http://localhost:3000

exporters:
  otlp:
    endpoint: jaeger:4317
    tls:
      insecure: true

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [otlp, logging]

export OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
export OTEL_EXPORTER_OTLP_PROTOCOL=grpc
export OTEL_EXPORTER_OTLP_TIMEOUT=10s
export OTEL_EXPORTER_OTLP_HEADERS="api-key=abc123"
export OTEL_EXPORTER_OTLP_COMPRESSION=gzip

export OTEL_TRACES_EXPORTER=otlp
export OTEL_METRICS_EXPORTER=otlp
export OTEL_LOGS_EXPORTER=otlp

OTel SDK
  ├── TracerProvider
  │     └── Tracer → Span
  ├── MeterProvider
  │     └── Meter → Instrument
  └── LoggerProvider
        └── Logger → LogRecord

所有 Provider 共享 Resource 标识

import (
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    sdkmetric "go.opentelemetry.io/otel/sdk/metric"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace"
    "go.opentelemetry.io/otel/exporters/otlp/otlpmetric"
)

// 公共 Resource
res := resource.NewWithAttributes(
    semconv.ServiceName("user-service"),
    semconv.DeploymentEnvironment("production"),
)

// TracerProvider
traceExporter, _ := otlptrace.New(ctx, otlptracegrpc.NewClient())
tp := sdktrace.NewTracerProvider(
    sdktrace.WithResource(res),
    sdktrace.WithBatcher(traceExporter),
    sdktrace.WithSampler(sdktrace.AlwaysSample()),
)
otel.SetTracerProvider(tp)

// MeterProvider
metricExporter, _ := otlpmetric.New(ctx, otlpmetricgrpc.NewClient())
mp := sdkmetric.NewMeterProvider(
    sdkmetric.WithResource(res),
    sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExporter)),
)
otel.SetMeterProvider(mp)

// 使用
tracer := otel.Tracer("api-server")
meter := otel.Meter("api-server")

初始化：
  Resource → TracerProvider + MeterProvider + LoggerProvider

运行时：
  TracerProvider → Tracer → Span
  MeterProvider → Meter → Instrument

关闭：
  LoggerProvider.Shutdown()
  MeterProvider.Shutdown()
  TracerProvider.Shutdown()

K8s Cluster
  │
  ├── Node 1
  │     └── otel-agent (DaemonSet)
  │           └── OTLP Export → otel-gateway
  │
  ├── Node 2
  │     └── otel-agent (DaemonSet)
  │           └── OTLP Export → otel-gateway
  │
  └── otel-gateway (Deployment)
        └── Export → Jaeger/Prometheus/OTLP

apiVersion: v1
kind: ConfigMap
metadata:
  name: otel-agent-conf
  namespace: observability
data:
  config.yaml: |
    receivers:
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
          http:
            endpoint: 0.0.0.0:4318

    processors:
      batch:
        timeout: 1s
        send_batch_size: 1024
      memory_limiter:
        check_interval: 1s
        limit_mib: 256

    exporters:
      otlp:
        endpoint: otel-gateway:4317
        tls:
          insecure: true

    service:
      pipelines:
        traces:
          receivers: [otlp]
          processors: [memory_limiter, batch]
          exporters: [otlp]    
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: otel-agent
spec:
  selector:
    matchLabels:
      app: otel-agent
  template:
    metadata:
      labels:
        app: otel-agent
    spec:
      containers:
        - name: otel-agent
          image: otel/opentelemetry-collector-contrib:0.100.0
          args: ["--config=/conf/config.yaml"]
          ports:
            - containerPort: 4317
              name: otlp-grpc
            - containerPort: 4318
              name: otlp-http
          resources:
            limits:
              memory: 512Mi
              cpu: 500m
            requests:
              memory: 128Mi
              cpu: 100m
          volumeMounts:
            - name: config
              mountPath: /conf
      volumes:
        - name: config
          configMap:
            name: otel-agent-conf
apiVersion: v1
kind: Service
metadata:
  name: otel-agent
spec:
  ports:
    - name: otlp-grpc
      port: 4317
    - name: otlp-http
      port: 4318
  selector:
    app: otel-agent

apiVersion: apps/v1
kind: Deployment
metadata:
  name: otel-gateway
spec:
  replicas: 2
  selector:
    matchLabels:
      app: otel-gateway
  template:
    spec:
      containers:
        - name: otel-gateway
          image: otel/opentelemetry-collector-contrib:0.100.0
          args: ["--config=/conf/config.yaml"]
          resources:
            limits:
              memory: 1Gi
              cpu: "2"
            requests:
              memory: 512Mi
              cpu: "1"
apiVersion: v1
kind: Service
metadata:
  name: otel-gateway
spec:
  ports:
    - name: otlp-grpc
      port: 4317
    - name: otlp-http
      port: 4318
  selector:
    app: otel-gateway

// Server Kind
span := tracer.Start(ctx, "HTTP GET /api/users",
    trace.WithSpanKind(trace.SpanKindServer),
)

// Client Kind
span := tracer.Start(ctx, "SELECT * FROM users",
    trace.WithSpanKind(trace.SpanKindClient),
)

// Internal Kind
span := tracer.Start(ctx, "process_data",
    trace.WithSpanKind(trace.SpanKindInternal),
)

// Producer Kind
span := tracer.Start(ctx, "publish order event",
    trace.WithSpanKind(trace.SpanKindProducer),
)

// Consumer Kind
span := tracer.Start(ctx, "consume order event",
    trace.WithSpanKind(trace.SpanKindConsumer),
)

Server ← Client ← Server ← Client
  │                    │
  └── Internal         └── Internal
       │                     │
       └── Internal          └── Internal

Producer（消息发送）
  → Trace 结束
  → Consumer（消息接收，可能在不同 Trace）
    → 通过 Link 关联

import (
    "go.opentelemetry.io/otel/codes"
    "go.opentelemetry.io/otel/attribute"
    "go.opentelemetry.io/otel/trace"
)

func processRequest(ctx context.Context, req Request) error {
    ctx, span := tracer.Start(ctx, "process_request")
    defer span.End()

    result, err := db.Query(ctx, req.UserID)
    if err != nil {
        // 方式1：设置 Span Status
        span.SetStatus(codes.Error, "database query failed")

        // 方式2：添加错误事件（包含堆栈）
        span.AddEvent("exception", trace.WithAttributes(
            attribute.String("exception.type", "DBError"),
            attribute.String("exception.message", err.Error()),
            attribute.String("exception.stacktrace", string(debug.Stack())),
        ))

        // 方式3：记录错误属性
        span.SetAttributes(
            attribute.Bool("error", true),
            attribute.String("error.type", "DatabaseError"),
        )

        return err
    }

    span.SetStatus(codes.Ok, "success")
    return nil
}

// 通用 HTTP Attributes
span.SetAttributes(
    semconv.HTTPMethod("GET"),
    semconv.HTTPURL("/api/users"),
    semconv.HTTPStatusCode(200),
    semconv.HTTPRequestContentLength(1024),
)

// HTTP Server Attributes
span.SetAttributes(
    semconv.NetHostName("api.example.com"),
    semconv.NetTransportTCP,
)

// HTTP Client Attributes
span.SetAttributes(
    semconv.HTTPTarget("/api/users"),
    semconv.PeerService("user-service"),
)

span.SetAttributes(
    semconv.DBSystemPostgreSQL,
    semconv.DBName("users"),
    semconv.DBStatement("SELECT * FROM users WHERE id = ?"),
    semconv.DBOperation("SELECT"),
    semconv.DBSQLTable("users"),
)

span.SetAttributes(
    semconv.MessagingSystemKafka,
    semconv.MessagingDestination("orders"),
    semconv.MessagingOperationProcess,
    semconv.MessagingMessageID("msg-123"),
)

res := resource.NewWithAttributes(
    semconv.SchemaURL,
    semconv.ServiceName("user-service"),
    semconv.ServiceVersion("1.2.3"),
    semconv.ServiceInstanceID("pod-abc"),
    semconv.DeploymentEnvironment("production"),
    semconv.CloudProviderAWS,
    semconv.CloudRegion("ap-southeast-1"),
    semconv.HostName("node-1"),
    semconv.ContainerID("docker-abc"),
)

```mermaid
graph TD
    A1["Receivers: otlp"] --> A2["Processors: batch"] --> A3["Exporters: spanmetrics"]
    A3 --> C["Connector: spanmetrics<br/>Traces → Metrics"]
    C --> B1["Receivers: spanmetrics"] --> B2["Processors: batch"] --> B3["Exporters: prometheus"]
    subgraph PA["Pipeline A (Traces)"]
        A1
        A2
        A3
    end
    subgraph PB["Pipeline B (Metrics)"]
        B1
        B2
        B3
    end

**Span Metrics Connector：**
```yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317

processors:
  batch:
    timeout: 1s
    send_batch_size: 1024

connectors:
  spanmetrics:
    # 从 Span 生成 Metrics 维度
    dimensions:
      - name: http.method
        default: GET
      - name: http.status_code
        default: 200
      - name: http.target

    # Histogram 配置
    latency_histogram_buckets:
      - 5ms
      - 10ms
      - 25ms
      - 50ms
      - 100ms
      - 250ms
      - 500ms
      - 1000ms
      - 2500ms
      - 5000ms

    # 指标命名
    metrics_flush_interval: 1m

exporters:
  prometheus:
    endpoint: 0.0.0.0:8889
  logging:
    verbosity: detailed

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [spanmetrics]
    metrics:
      receivers: [spanmetrics]
      processors: [batch]
      exporters: [prometheus, logging]

# Span 统计生成以下 Prometheus 指标：
traces_span_count_total{dimension="value"}
traces_latency_bucket{dimension="value",le="0.005"}
traces_latency_bucket{dimension="value",le="0.01"}
traces_latency_count{dimension="value"}
traces_latency_sum{dimension="value"}

# Jaeger 启动时启用 OTLP
services:
  jaeger:
    image: jaegertracing/all-in-one:latest
    ports:
      - "16686:16686"  # UI
      - "4317:4317"    # OTLP gRPC
      - "4318:4318"    # OTLP HTTP

# Collector 配置
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317

processors:
  batch:
    timeout: 1s
    send_batch_size: 1024

exporters:
  jaeger:
    endpoint: jaeger:14250
    tls:
      insecure: true

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [jaeger]

exporter, _ := otlptracegrpc.New(ctx,
    otlptracegrpc.WithEndpoint("jaeger:4317"),
    otlptracegrpc.WithInsecure(),
)

tp := sdktrace.NewTracerProvider(
    sdktrace.WithBatcher(exporter),
    sdktrace.WithResource(res),
)
otel.SetTracerProvider(tp)

processors:
  attributes:
    actions:
      # 1. 插入固定属性
      - key: environment
        value: production
        action: insert

      # 2. 更新服务名
      - key: service.name
        value: api-gateway
        action: upsert

      # 3. 删除敏感属性
      - key: auth.token
        action: delete
      - key: password
        action: delete

      # 4. Hash IP（匿名化）
      - key: client.ip
        action: hash

      # 5. 提取路径（从 URL 中提取路径模式）
      - key: http.target
        pattern: ^/([^/?]+)
        action: extract

  # 条件处理
  attributes/conditional:
    include:
      match_type: regexp
      attributes:
        - key: http.status_code
          value: "5[0-9]{2}"
    actions:
      - key: error
        value: true
        action: upsert

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [attributes, batch]
      exporters: [otlp]

# 基础配置
export OTEL_SERVICE_NAME=user-service
export OTEL_RESOURCE_ATTRIBUTES="deployment.environment=production,service.version=1.2.3"
export OTEL_PROPAGATORS=tracecontext,baggage

# OTLP 配置
export OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
export OTEL_EXPORTER_OTLP_HEADERS="api-key=abc123"
export OTEL_EXPORTER_OTLP_COMPRESSION=gzip

# 采样
export OTEL_TRACES_SAMPLER=parentbased_traceidratio
export OTEL_TRACES_SAMPLER_ARG=0.1

# 启动应用
java -jar app.jar

高吞吐量 → OTel SDK
    │
  BatchSpanProcessor
    ├── MaxQueueSize: 2048（默认）
    ├── BatchTimeout: 1s
    └── MaxExportBatchSize: 512
    │
  （队列满时的降级行为）
    ├── blocking: 阻塞应用（默认，高可靠）
    ├── dropping: 丢弃新 Span（实时性优先）
    └── 可通过 SDK 配置

// 高吞吐场景配置
bsp := sdktrace.NewBatchSpanProcessor(
    exporter,
    sdktrace.WithMaxQueueSize(4096),         // 增大队列
    sdktrace.WithMaxExportBatchSize(1024),    // 增大批量
    sdktrace.WithBatchTimeout(500*time.Millisecond), // 降低延迟
    sdktrace.WithExportTimeout(30*time.Second),
)

Collector 侧背压：

Receiver → Processors → Exporter
    │
  Memory Limiter
    ├── 内存 > limit → 返回 ResourceExhausted
    └── SDK 收到错误 → 重试或降级

SDK 侧背压：
  Exporter 失败 → BatchSpanProcessor 重试
  → 如果持续失败 → 队列满 → 阻塞应用

graph TD
    LB[Load Balancer] --> C1[Collector-1]
    LB --> C2[Collector-2]
    LB --> C3[Collector-3]
    C1 --> Exporters[Exporters - 本身 HA]
    C2 --> Exporters
    C3 --> Exporters

apiVersion: apps/v1
kind: Deployment
metadata:
  name: otel-gateway
spec:
  replicas: 3
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0  # 零停机更新
  selector:
    matchLabels:
      app: otel-gateway
  template:
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              podAffinityTerm:
                labelSelector:
                  matchLabels:
                    app: otel-gateway
                topologyKey: kubernetes.io/hostname
      containers:
        - name: collector
          image: otel/opentelemetry-collector-contrib:0.100.0
          readinessProbe:
            httpGet:
              path: /
              port: 13133
          livenessProbe:
            httpGet:
              path: /health
              port: 13133
          resources:
            limits:
              memory: 1Gi
              cpu: "2"
            requests:
              memory: 512Mi
              cpu: "1"

extensions:
  health_check:
    endpoint: 0.0.0.0:13133
  pprof:
    endpoint: 0.0.0.0:1777

service:
  extensions: [health_check, pprof]
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [otlp]

OTel API v1.x.x → 稳定，不向后兼容性破坏
OTel SDK v1.x.x → 稳定，遵循 API 规范
OTel Contrib v0.x.x → 实验性（Instrumentation）

规则：
  API v1.x ↔ SDK v1.x 兼容
  Contrib v0.x ↔ SDK v1.x 兼容
  API v1.x ↔ Contrib v0.x 兼容（通过 SDK）

1. API 和 SDK 版本必须匹配（v1.x 对 v1.x）
2. Instrumentation 版本独立，但要求 SDK 兼容版本
3. 跨语言服务要求 Exporters 版本兼容
4. Collector 版本独立于 SDK 版本

推荐：保持各组件版本同步更新

Histogram Metric (request.duration)
  Bucket: 0-100ms
    DataPoint: value=45ms
      Exemplar: {
        trace_id: "abc...",
        span_id: "123...",
        value: 45ms,
        timestamp: ...
      }
  Bucket: 100-500ms
    DataPoint: value=250ms
      Exemplar: {
        trace_id: "def...",
        span_id: "456...",
        value: 250ms,
        timestamp: ...
      }

processors:
  batch:
    exemplar:
      enabled: true
      max_exemplars: 20
      filter:
        - http.status_code: "5[0-9]{2}"

import (
    "go.opentelemetry.io/otel/sdk/metric"
    "go.opentelemetry.io/otel/sdk/metric/metricdata"
)

// 启用 Exemplar
provider := metric.NewMeterProvider(
    metric.WithReader(metric.NewPeriodicReader(
        exporter,
        metric.WithInterval(15*time.Second),
    )),
    metric.WithExemplarFilter(
        metric.WithExemplarFilter(
            metric.ExemplarFilterAlwaysOn,
        ),
    ),
)

# Prometheus v2.40+ 支持 Exemplar
# 通过 OTel Collector 导出 Exemplar

exporters:
  prometheus:
    endpoint: 0.0.0.0:8889
    enable_open_metrics: true   # 启用 Exemplar 导出

服务 → Agent (DaemonSet) → Gateway (Deployment) → 后端

每个环节的设计要点：

# 资源
Memory: 256MB  limit
CPU: 500m limit

# 处理
Memory Limiter: 200MB → 开始降级
Batch: 1s / 1024

# 采样（减轻后端压力）
Tail Sampling:
  Error: 100%
  Latency > 5s: 100%
  Others: 10%

# 高可用
Replicas: 3
Pod Anti-Affinity

# 处理
Memory Limiter: 1GB
Load Balancer

# 路由
多 Exporter → Jaeger + Prometheus + S3 归档

# 认证
API Key 验证 → 仅接受已知 Agent

Agent ↔ Gateway：TLS + API Key
Gateway ↔ Backend：mTLS（双向认证）
配置管理：K8s Secret 管理凭证
网络隔离：专用 Service Mesh 策略

// 所有服务使用相同的 Propagator 配置
import (
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/propagation"
)

func init() {
    otel.SetTextMapPropagator(
        propagation.NewCompositeTextMapPropagator(
            propagation.TraceContext{},
            propagation.Baggage{},
        ),
    )
}

HTTP: {HTTP Method} {Route}
  ✅ "GET /api/users/{id}"
  ✅ "POST /api/orders"
  ❌ "GET /api/users/123"   （包含具体 ID）

gRPC: {Service}/{Method}
  ✅ "UserService/GetUser"

DB: {DB Type} {Operation} {Table}
  ✅ "PostgreSQL SELECT users"

MQ: {Topic} {Operation}
  ✅ "orders PUBLISH"
  ✅ "orders CONSUME"

// HTTP 服务端必备
span.SetAttributes(
    semconv.HTTPMethod(r.Method),
    semconv.HTTPURL(r.URL.String()),
    semconv.HTTPStatusCode(statusCode),
    semconv.HTTPTarget(r.URL.Path),
    semconv.HTTPRequestContentLength(req.ContentLength),
    semconv.HTTPRoute(pattern),  // 路由模板
)

// 客户端必备
span.SetAttributes(
    semconv.HTTPMethod(method),
    semconv.HTTPURL(url),
    semconv.PeerService(serviceName),
)

Gateway 层采样（统一控制）：
  - 错误请求: 100%
  - 新功能灰度: 100%
  - P99+ 延迟: 100%
  - 基础: 10% 概率

Agent 层采样（预过滤）：
  - 健康检查: 0%
  - 批量处理: 1%

graph TD
    O["OTLP Input<br/>统一入口"] --> AUTH["Auth Extensions<br/>提取 tenant"]
    AUTH --> AP["Attributes Processor<br/>注入 tenant 属性"]
    AP --> TS["Tail Sampling Processor<br/>租户独立策略"]
    TS --> TA["Tenant A<br/>Exporter"]
    TS --> TB["Tenant B<br/>Exporter"]
    TS --> TC["Tenant C<br/>Exporter"]

extensions:
  oidc_auth:
    issuer_url: https://auth.example.com
    audience: otel-collector

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
        auth:
          authenticator: oidc_auth

processors:
  attributes/tenant:
    actions:
      - key: tenant_id
        from_context: auth.tenant_id
        action: upsert

  groupbyattrs:
    keys:
      - tenant_id

exporters:
  otlp/tenant_a:
    endpoint: tenant-a-backend:4317
    headers:
      authorization: Bearer ${TENANT_A_TOKEN}

  otlp/tenant_b:
    endpoint: tenant-b-backend:4317
    headers:
      authorization: Bearer ${TENANT_B_TOKEN}

  routing:
    from_attribute: tenant_id
    default_exporters: [logging]
    table:
      - value: tenant_a
        exporters: [otlp/tenant_a]
      - value: tenant_b
        exporters: [otlp/tenant_b]

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [attributes/tenant, groupbyattrs, routing]
      exporters: [routing]

Trace ID: 0af7651916cd43dd8448eb211c80319c
    │
  Hash(Trace ID) → 0-16383 整数
    │
  threshold = sampling_percentage × 16383 / 100
    │
  hash < threshold → 保留
  hash ≥ threshold → 丢弃

processors:
  probabilistic_sampler:
    # 采样百分比
    sampling_percentage: 10

    # 哈希种子（可选）
    hash_seed: 42

    # Attribute 来源（默认 trace_id）
    attribute_source: trace_id

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [probabilistic_sampler]
      exporters: [otlp]

Trace ID = abc123
  Service A: hash(abc123) = 500  → threshold(10%) = 1638 → 保留 ✓
  Service B: hash(abc123) = 500  → threshold(10%) = 1638 → 保留 ✓
  Service C: hash(abc123) = 500  → threshold(10%) = 1638 → 保留 ✓

同一个 Trace 在不同服务采样结果一致

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
        # gRPC 特有参数
        max_recv_msg_size_mib: 10
        max_concurrent_streams: 100
        read_buffer_size: 512KB
        write_buffer_size: 512KB

receivers:
  otlp:
    protocols:
      http:
        endpoint: 0.0.0.0:4318
        cors:
          allowed_origins:
            - http://localhost:3000
          allowed_headers:
            - Content-Type
            - Authorization
        max_request_body_size: 2MB

服务间通信（Collector → Backend）：gRPC
  - 高性能二进制传输
  - 流式处理支持
  - 双向 TLS 认证

边缘采集（SDK → Collector）：HTTP
  - 调试方便（curl）
  - 浏览器支持
  - 负载均衡简单

浏览器/移动端：HTTP
  - 原生 Web 协议
  - 避免 gRPC-Web 依赖

processors:
  k8sattributes:
    # K8s API 配置
    auth_type: serviceAccount
    passthrough: false

    # 提取 Pod 信息
    extract:
      metadata:
        - k8s.pod.name
        - k8s.pod.uid
        - k8s.namespace.name
        - k8s.node.name
        - k8s.pod.start_time

      # 提取 Labels
      labels:
        - tag_name: app.label
          key: app
          from: pod

      # 查找 Owner
      owner_lookup: true

    # 命名空间过滤
    filter:
      namespaces:
        - production
        - staging
      node_from_env_var: K8S_NODE_NAME

    # Pod 关联
    pod_association:
      - sources:
          - from: resource_attribute
            name: k8s.pod.uid
      - sources:
          - from: connection

原始 Span：
  resource: {ip: "10.0.1.5"}

K8s Processor 处理后：
  resource: {
    ip: "10.0.1.5",
    k8s.pod.name: "api-server-7d9f8c-abc12",
    k8s.namespace.name: "production",
    k8s.node.name: "node-3",
    k8s.deployment.name: "api-server",
    app.label: "api-server",
    k8s.pod.uid: "pod-uid-xxx"
  }