SkyWalking 面试题

graph TD
    A["Application"] --> B["Agent<br/>SkyWalking Agent<br/>gRPC/HTTP"]
    B --> C["OAP Server<br/>集群"]
    C --> D["Storage<br/>ES/MySQL"]
    C --> E["Alarm<br/>告警"]
    D -->|"GraphQL API"| F["UI"]

# Java Agent（无侵入）
java -javaagent:/path/to/skywalking-agent.jar \
     -Dskywalking.agent.service_name=user-service \
     -Dskywalking.collector.backend_service=oap:11800 \
     -jar app.jar

JVM 启动
  │
  javaagent 参数指定 SkyWalking Agent
  │
  Agent 初始化
    ├── 解析配置 (agent.config)
    ├── 加载插件
    ├── Instrumentation (字节码增强)
    └── 连接 OAP Server
  │
  运行时
    ├── 拦截框架方法 (HTTP/gRPC/DB/MQ)
    ├── 创建/传播 Span
    ├── 收集 Trace/Metrics
    └── 异步发送到 OAP

// Agent 插件的拦截点定义（以 Tomcat 插件为例）
public class TomcatInterceptor implements InstanceMethodsAroundInterceptor {
    @Override
    public void beforeMethod(EnhancedInstance objInst, Method method,
                             Object[] allArguments, Class<?>[] argumentsTypes) {
        // 请求入口：创建 EntrySpan
        Span span = ContextManager.createEntrySpan(
            request.getRequestURI(),
            new ContextCarrier().deserialize(
                request.getHeader("sw8")  // SkyWalking 协议头
            )
        );
        span.setComponent(ComponentsDefine.TOMCAT);
    }

    @Override
    public Object afterMethod(EnhancedInstance objInst, Method method,
                              Object[] allArguments, Class<?>[] argumentsTypes,
                              Object ret) {
        // 请求结束：结束 Span
        ContextManager.stopSpan();
        return ret;
    }
}

无需修改业务代码
无需额外 SDK 依赖
支持热插拔（运行时不上报）
升级 Agent 只需替换 jar 包

Trace（全局调用链）
  │
  ├── Segment 1 (服务A: user-service)
  │     ├── Span 1 (Entry: HTTP GET /api/users)
  │     │     └── SpanRef → Segment 2
  │     ├── Span 2 (Local: validateToken)
  │     └── Span 3 (Exit: JDBC SELECT users)
  │           └── SpanRef → DB 实例
  │
  └── Segment 2 (服务B: order-service)
        ├── Span 1 (Entry: gRPC createOrder)
        └── Span 2 (Exit: Kafka send)

// Segment (服务级别)
{
    "traceSegmentId": "uuid",
    "serviceId": "user-service-id",
    "serviceInstanceId": "instance-1",
    "spans": [Span, Span, ...]
}

// Span (操作级别)
{
    "spanId": 0,
    "parentSpanId": -1,       // -1 表示根 Span
    "startTime": 1716700000000,
    "endTime": 1716700000500,
    "operationName": "GET /api/users",
    "spanType": "Entry",      // Entry / Local / Exit
    "spanLayer": "Http",
    "componentId": 1,         // Tomcat
    "isError": false,
    "logs": [],
    "tags": [],
    "refs": [SpanRef]         // 跨进程引用
}

Entry Span (Tomcat: HTTP GET /api/orders)
    │
    ├── Local Span (validatePermission)
    │
    ├── Local Span (checkCache)
    │
    └── Exit Span (JDBC: SELECT orders)
         │
         └── [远程服务 Entry Span]

         Exit Span (gRPC: UserService/GetUser)
              │
              └── [用户服务 Entry Span]

// Entry Span（服务入口）
Span entrySpan = ContextManager.createEntrySpan(
    request.getRequestURI(),
    carrier  // 跨进程传播上下文
);
entrySpan.setComponent(ComponentsDefine.TOMCAT);

// Local Span（本地操作）
Span localSpan = ContextManager.createLocalSpan(
    "validatePermission"
);
localSpan.setTag("role", "admin");

// Exit Span（远程调用）
Span exitSpan = ContextManager.createExitSpan(
    "/UserService/GetUser",
    carrier,
    "user-service:8080"
);
exitSpan.setComponent(ComponentsDefine.GRPC);

graph TD
    LB[Nginx/LB] --> OAP1[OAP-1]
    LB --> OAP2[OAP-2]
    LB --> OAP3[OAP-3]
    OAP1 --> CC[Cluster Coordinator - Nacos/ZK/K8s]
    OAP2 --> CC
    OAP3 --> CC

# application.yml
cluster:
  selector: ${SW_CLUSTER:zookeeper}
  # Nacos Cluster
  nacos:
    serviceName: ${SW_CLUSTER_NACOS_SERVICE_NAME:SkyWalking_OAP_Cluster}
    hostPort: ${SW_CLUSTER_NACOS_HOST_PORT:localhost:8848}

  # Kubernetes Cluster
  kubernetes:
    watchTimeoutSeconds: ${SW_CLUSTER_K8S_WATCH_TIMEOUT:60}
    namespace: ${SW_CLUSTER_K8S_NAMESPACE:default}
    labelSelector: ${SW_CLUSTER_K8S_LABEL:app=oap}
    uidEnvName: ${SW_CLUSTER_K8S_UID:SKYWALKING_COLLECTOR_UID}

# ES 存储
storage:
  selector: ${SW_STORAGE:elasticsearch}
  elasticsearch:
    clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200}
    indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:2}
    indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0}
    bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:1000}
    bulkSize: ${SW_STORAGE_ES_BULK_SIZE:20}
    flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10}
    concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2}
    advanced: ${SW_STORAGE_ES_ADVANCED:""}

# MySQL 存储
storage:
  selector: ${SW_STORAGE:mysql}
  mysql:
    properties:
      jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:3306/swtest"}
      dataSource.user: ${SW_DATA_SOURCE_USER:root}
      dataSource.password: ${SW_DATA_SOURCE_PASSWORD:root}
    metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000}

# TiDB 存储
storage:
  selector: ${SW_STORAGE:tidb}
  tidb:
    properties:
      jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://tidb-server:4000/skywalking"}
      dataSource.user: ${SW_DATA_SOURCE_USER:root}
    maxSizeOfArrayColumn: ${SW_STORAGE_MAX_SIZE_OF_ARRAY_COLUMN:500}

# alarm-settings.yml
rules:
  # 规则名称
  service_resp_time_rule:
    # 指标名称
    metrics-name: service_resp_time
    # 阈值
    threshold: 1000
    # 操作符：> / < / =
    op: ">"
    # 评估周期（分钟）
    period: 10
    # 持续次数
    count: 3
    # 静默时间（分钟）
    silence-period: 5
    # 消息模板
    message: |
      Service {name} response time {value} exceeds 1000ms in last 10 minutes.      

  service_sla_rule:
    metrics-name: service_sla
    threshold: 80
    op: "<"
    period: 10
    count: 3
    message: |
      Service {name} SLA {value} is lower than 80% in last 10 minutes.      

  # 端点延迟告警
  endpoint_resp_time_rule:
    metrics-name: endpoint_resp_time
    threshold: 2000
    op: ">"
    period: 10
    count: 5
    message: |
      Endpoint {name} response time {value} exceeds 2000ms.      

  # 异常率告警
  endpoint_sla_rule:
    metrics-name: endpoint_sla
    threshold: 90
    op: "<"
    period: 10
    count: 3
    message: |
      Endpoint {name} SLA {value} is lower than 90%.      

# 通知方式
hooks:
  # Webhook
  webhook:
    default:
      is-default: true
      urls:
        - http://alertmanager:9093/api/v1/alerts
        - http://webhook-service:8080/skywalking-alert

  # 钉钉
  dingtalk:
    text-template: |
      SkyWalking Alarm:
      - Rule: {ruleName}
      - Service: {scopeName}
      - Message: {message}
      - Time: {date}      
    webhooks:
      - url: https://oapi.dingtalk.com/robot/send?access_token=xxx
        secret: your-secret

  # 企业微信
  wechat:
    webhooks:
      - url: https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=xxx

  # Slack
  slack:
    text-template: |
      *SkyWalking Alarm*
      *Rule:* {ruleName}
      *Service:* {scopeName}
      *Message:* {message}      
    webhooks:
      - url: https://hooks.slack.com/services/xxx

Agent 拦截每次远程调用
    │
  Entry Span 记录服务A
  Exit Span 记录服务B
    │
  SpanRef (跨进程引用)
    │
  OAP Server 聚合
    │
  服务依赖关系：
  ServiceA → ServiceB: 1000次/分钟, P50=50ms, 错误率=1%

// Exit Span 生成的引用
SpanRef ref = new SpanRef();
ref.setParentService("user-service");
ref.setParentInstance("user-service-1");
ref.setParentEndpoint("GET /api/users");
ref.setTargetService("order-service");
ref.setTargetEndpoint("gRPC /OrderService/Create");
ref.setRefType(RefType.CrossProcess);

{
    "nodes": [
        {
            "id": "user-service",
            "name": "user-service",
            "type": "SERVICE",
            "isReal": true,
            "metrics": {
                "avgResponseTime": 150,
                "cpm": 1000,      // 每分钟请求数
                "apdex": 0.95,    // 应用性能指数
                "sla": 99.5       // 服务等级协议
            }
        }
    ],
    "calls": [
        {
            "source": "user-service",
            "target": "order-service",
            "metrics": {
                "avgResponseTime": 50,
                "cpm": 800,
                "sla": 99.8
            }
        }
    ]
}

拓扑图展示：
  - 服务节点（圆角矩形）
  - 依赖连线（箭头指向）
  - 节点颜色表示健康状态（绿/黄/红）
  - 连线上显示调用量和延迟
  - 点击节点查看详情
  - 拖动布局

故障场景拓扑：
  服务变红 → 箭头变红 → 调用方 SLA 下降
  → 快速定位故障影响范围

# 完整的 Java Agent 支持
# 框架：Spring Boot, Dubbo, Tomcat, gRPC, Kafka, JDBC, Redis, RocketMQ
# 功能：自动 Trace、自动 Metrics、自动告警
java -javaagent:/skywalking-agent/skywalking-agent.jar \
     -Dskywalking.agent.service_name=user-service \
     -Dskywalking.collector.backend_service=oap:11800 \
     -jar app.jar

import (
    "github.com/apache/skywalking-go"
)

func main() {
    // 自动配置（通过环境变量）
    // SW_AGENT_NAME=user-service
    // SW_AGENT_COLLECTOR_BACKEND_SERVICES=oap:11800

    // 手动创建 Span
    tracer, _ := skywalking.NewTracer("user-service")
    span := tracer.StartSpan("/api/users")
    defer span.End()

    // 自动埋点（HTTP 框架自动拦截）
    http.HandleFunc("/api/users", handler)
}

# agent/config/agent.config

# gRPC（默认，推荐）
collector.backend_service=${SW_AGENT_COLLECTOR_BACKEND_SERVICES:oap:11800}

# HTTP（备选）
# collector.backend_service=${SW_AGENT_COLLECTOR_BACKEND_SERVICES:oap:12800}

# application.yml
core:
  gRPCHost: ${SW_GRPC_HOST:0.0.0.0}
  gRPCPort: ${SW_GRPC_PORT:11800}

receiver-sharing:
  selector: ${SW_RECEIVER_SHARING:default}
  default:
    restHost: ${SW_REST_HOST:0.0.0.0}
    restPort: ${SW_REST_PORT:12800}
    restContextPath: ${SW_REST_CONTEXT_PATH:/}

方式一：OTel Collector → SkyWalking OAP
  App → OTel SDK → OTel Collector → OAP (gRPC)
                                        │
                                  Analysis & Storage

方式二：OTel SDK → SkyWalking OAP（直接）
  App → OTel SDK (SkyWalking Exporter) → OAP

方式三：SkyWalking Agent → OTel Collector
  App → SW Agent → OTelemetryOAP → OTel Collector → Any Backend

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317

processors:
  batch:
    timeout: 1s
    send_batch_size: 1024

exporters:
  skywalking:
    endpoint: skywalking-oap:11800
    num_grpc_workers: 4

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [skywalking]

多语言环境：Java 用 SW Agent，其他语言用 OTel SDK
 → OTel Collector 统一转发到 SkyWalking

已有 OTel 基础设施：
 → OTel Collector added SW Exporter → 同时发送到 SW 和 Jaeger

service_resp_time   : 服务平均响应时间 (ms)
service_cpm         : 每分钟调用次数
service_sla         : 服务成功率 (%)
service_apdex       : 应用性能指数 (0-1)

instance_jvm_cpu               : CPU 使用率 (%)
instance_jvm_memory_heap       : 堆内存 (bytes)
instance_jvm_memory_noheap     : 非堆内存 (bytes)
instance_jvm_gc_time            : GC 耗时 (ms)
instance_jvm_gc_count           : GC 次数
instance_jvm_thread_count       : 线程数
instance_jvm_thread_blocked     : 阻塞线程数

endpoint_resp_time  : 端点平均响应时间 (ms)
endpoint_cpm        : 端点每分钟调用次数
endpoint_sla        : 端点成功率 (%)
endpoint_avg        : 平均延迟 (ms)
endpoint_p50/p90/p95/p99 : 百分位延迟

# SkyWalking OAP 暴露 Prometheus 格式指标
telemetry:
  selector: ${SW_TELEMETRY:prometheus}
  prometheus:
    host: ${SW_TELEMETRY_PROMETHEUS_HOST:0.0.0.0}
    port: ${SW_TELEMETRY_PROMETHEUS_PORT:1234}
    sslEnabled: ${SW_TELEMETRY_PROMETHEUS_SSL_ENABLED:false}

应用 → JDBC Driver → SkyWalking 插件
                          │
                    拦截 PreparedStatement.execute()
                          │
                    创建 Exit Span (DB 类型)
                          │
                    记录 SQL 语句、参数
                          │
                    计算执行耗时
                          │
                    标记慢查询（配置阈值）
                          │
                    发送到 OAP Server

# agent.config
# 慢查询阈值（默认 200ms）
plugin.mysql.trace_sql_parameters=true
plugin.mysql.sql_parameters_max_length=256

# PostgreSQL
plugin.postgresql.trace_sql_parameters=true

{
    "traceId": "abc123",
    "statement": "SELECT * FROM users WHERE status = ? AND created_at > ?",
    "parameters": ["active", "2026-01-01"],
    "latency": 3500,
    "startTime": "2026-05-26T10:00:00Z",
    "service": "user-service",
    "instance": "user-service-1",
    "database": "postgresql://db-host:5432/users",
    "connection": "100"
}

SkyWalking UI → Database → Slow SQL
  显示：SQL 语句、执行次数、平均延迟、最大延迟
  排序：按延迟降序
  过滤：按数据库类型、服务名

GraphQL API：
query slowSQLs {
    slowSQLs(condition: {
        metricName: "database_slow_sql"
        serviceId: "user-service"
    }) {
        statement
        latency
        count
    }
}

sw8: 1-{traceId}-{segmentId}-{spanId}-{service}-{instance}-{endpoint}-{targetAddress}-{correlation}

服务A (user-service)
  创建 Exit Span → 设置 sw8 Header
  HTTP 请求 → sw8: 1-abc123-segA-0-user-service-user-1-GET%2Fapi%2Fusers-10.0.1.5:8080-

服务B (order-service)
  HTTP 接收 → 解析 sw8 Header
  创建 Entry Span → 继承上下文
  创建新的 Segment

// Kafka 消息的 Context 传播通过消息 Header
ProducerRecord<String, String> record = new ProducerRecord<>("orders", message);

// SkyWalking 自动注入到 record.headers()
// headers: ["sw8": "1-abc123-segA-0-user-service-user-1-sendOrder-..."]

# agent/config/agent.config

# 基于 Trace 的采样率（默认 100%）
agent.sample_n_per_3_secs=${SW_AGENT_SAMPLE_N_PER_3_SECS:-1}
# -1: 全部采样
# 正整数: 每 3 秒采样 N 条 Trace

# 忽略指定端点
agent.ignore_suffix=${SW_AGENT_IGNORE_SUFFIX:.jpg,.jpeg,.js,.css,.png,.bmp,.ico,.gif}

# 忽略指定路径
agent.operation_name_threshold: ${SW_AGENT_OPERATION_NAME_THRESHOLD:150}

# application.yml
core:
  # Trace 采样率（0-10000，精确到万分比）
  sampleRate: ${SW_CORE_SAMPLE_RATE:10000}

  # 慢查询采样率
  slowTraceThreshold: ${SW_SLOW_TRACE_THRESHOLD:-1}

# ES 存储 TTL 配置
storage:
  elasticsearch:
    # Trace 数据
    recordDataTTL: ${SW_STORAGE_ES_RECORD_DATA_TTL:7}
    # 分钟级指标
    minuteMetricsDataTTL: ${SW_STORAGE_ES_MINUTE_METRICS_DATA_TTL:7}
    # 小时级指标
    hourMetricsDataTTL: ${SW_STORAGE_ES_HOUR_METRICS_DATA_TTL:30}
    # 天级指标
    dayMetricsDataTTL: ${SW_STORAGE_ES_DAY_METRICS_DATA_TTL:90}
    # 月份级指标
    monthMetricsDataTTL: ${SW_STORAGE_ES_MONTH_METRICS_DATA_TTL:18}

Services Dashboard:
  - 全局响应时间趋势
  - 全局吞吐量趋势
  - 全局 SLA 趋势
  - 慢端点 Top 10
  - 错误端点 Top 10
  - 服务列表（SLA 排序）

Instance Dashboard:
  - JVM CPU 使用率
  - JVM 堆/非堆内存
  - GC 频率和耗时
  - 实例级吞吐量
  - 实例级延迟

Endpoint Dashboard:
  - 端点延迟 (P50/P90/P95/P99)
  - 端点吞吐量
  - 端点 SLA
  - 慢端点详情

查询条件：
  - 服务名 / 端点名
  - 时间范围
  - Trace ID (精确查询)
  - 最小/最大耗时
  - 是否错误
  - Tags 过滤

列表视图：
  Trace ID / 端点 / 耗时 / 状态 / 时间
  ✓ 按耗时降序（慢请求优先）

详情视图：
  甘特图展示 Span 时间线
  每个 Span 包含 Tags / Logs / 关联
  ✓ Span 展开查看属性
  ✓ 跨服务跳转

SkyWalking Agent
    │
  注入 Trace ID 到 MDC
    │
  Logback/Log4j 配置 %X{tid}
    │
  日志输出含 traceId
    │
  在 UI 中可点击跳转

<configuration>
    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <!-- %tid 由 SkyWalking Agent 自动替换为 Trace ID -->
            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} [%tid] - %msg%n</pattern>
        </encoder>
    </appender>

    <!-- SkyWalking gRPC Log Reporter -->
    <appender name="GRPC" class="org.apache.skywalking.apm.toolkit.log.logback.v1.x.log.GRPCLogClientAppender">
        <encoder>
            <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} [%tid] - %msg%n</pattern>
        </encoder>
    </appender>

    <root level="INFO">
        <appender-ref ref="CONSOLE"/>
        <appender-ref ref="GRPC"/>
    </root>
</configuration>

<Configuration>
    <Appenders>
        <Console name="Console" target="SYSTEM_OUT">
            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} [%X{tid}] - %msg%n"/>
        </Console>
        <GRPCLogClientAppender name="GRPC">
            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} [%X{tid}] - %msg%n"/>
        </GRPCLogClientAppender>
    </Appenders>

    <Loggers>
        <Root level="info">
            <AppenderRef ref="Console"/>
            <AppenderRef ref="GRPC"/>
        </Root>
    </Loggers>
</Configuration>

<dependency>
    <groupId>org.apache.skywalking</groupId>
    <artifactId>apm-toolkit-logback-1.x</artifactId>
    <version>${skywalking.version}</version>
</dependency>

1. 选择目标端点 → 发起 Profile 任务
2. Agent 按配置频率采样线程堆栈
3. 堆栈数据 → OAP 聚合分析
4. UI 展示火焰图 / 调用树
5. 定位热点方法

# agent/config/agent.config

# 最大采样线程数
plugin.threading.max_sampling_count: ${SW_PLUGIN_THREADING_MAX_SAMPLING_COUNT:20}

# Profile 线程池大小
plugin.threading.thread_pool_size: ${SW_PLUGIN_THREADING_THREAD_POOL_SIZE:4}

# 采样持续时间（分钟）
plugin.threading.duration_based_sampling: ${SW_PLUGIN_THREADING_DURATION_BASED_SAMPLING:true}

参数说明：
  - 端点: 要分析的 API
  - 监控时长: 1-10 分钟
  - 采样间隔: 10-100ms
  - 最大采样数: 100-10000

监控过程中：
  Agent 按间隔采集线程堆栈
  堆栈 trace → OAP 分析
  → 生成调用树和火焰图

火焰图（Flame Graph）：
  X 轴: 代码调用栈宽度（越宽 = 调用次数越多）
  Y 轴: 调用深度（越深 = 调用链越长）
  颜色: 无特殊含义（随机分配）

调用树（Call Tree）：
  Root 方法
    ├── 子方法 1 (耗时 50% / 调用 100 次)
    ├── 子方法 2 (耗时 30% / 调用 50 次)
    │     ├── 子方法 2.1 (耗时 20%)
    │     └── 子方法 2.2 (耗时 10%)
    └── 子方法 3 (耗时 20%)

graph TD
    Envoy[Istio Sidecar - Envoy] --> W1[方式一: Istio Telemetry API v2]
    W1 --> Wasm[Wasm Plugin]
    Wasm --> SW[SkyWalking]
    Envoy --> W2[方式二: Envoy Access Log + Metadata]
    W2 --> SW2[SkyWalking]

# OAP 配置启用 Istio 数据接收
receiver-envoy:
  selector: ${SW_RECEIVER_ENVOY:default}
  default:
    acceptMetricsService: ${SW_RECEIVER_ENVOY_ACCEPT_METRICS_SERVICE:true}
    acceptTracingService: ${SW_RECEIVER_ENVOY_ACCEPT_TRACING_INFORMATION:true}

# ALS (Access Log Service) 配置
envoy-metrics:
  alsHTTPAnalysis: ${SW_ENVOY_METRICS_ALS_HTTP_ANALYSIS:""}

apiVersion: telemetry.istio.io/v1alpha1
kind: Telemetry
metadata:
  name: skywalking
  namespace: istio-system
spec:
  accessLogging:
    - providers:
        - name: skywalking
  tracing:
    - providers:
        - name: skywalking
apiVersion: extensions.istio.io/v1alpha1
kind: WasmPlugin
metadata:
  name: skywalking
  namespace: istio-system
spec:
  selector:
    matchLabels:
      istio: sidecar-injector
  url: oci://skywalking-wasm/skywalking-envoy:latest
  phase: STATS
  priority: 100

应用拓扑（Agent 采集）：
  - 在应用代码层面构建
  - 包含应用框架信息
  - 适用于有 Agent 部署的服务

Mesh 拓扑（Istio 采集）：
  - 在网络层面构建
  - 无需 Agent，容器化即可
  - 适用于所有 Sidecar 服务

两者可同时使用，互相补充

# SkyWalking 告警规则支持的指标前缀
# service_* : 服务级别指标
# endpoint_* : 端点级别指标
# instance_* : 实例级别指标
# database_* : 数据库调用指标

rules:
  service_resp_time_rule:
    metrics-name: service_resp_time
    threshold: 1000
    op: ">"
    period: 10
    count: 3

  service_sla_rule:
    metrics-name: service_sla
    threshold: 80
    op: "<"
    period: 10
    count: 3

# Prometheus 等效告警规则
groups:
  - name: skywalking-equivalent
    rules:
      - alert: ServiceResponseTimeHigh
        expr: skywalking_service_resp_time > 1000
        for: 10m
        annotations:
          summary: "Service XQOPEN $labels.service XQCLOSE response time > 1000ms"

      - alert: ServiceSLALow
        expr: skywalking_service_sla < 80
        for: 10m
        annotations:
          summary: "Service XQOPEN $labels.service XQCLOSE SLA < 80%"

# SkyWalking 告警 → Prometheus AlertManager
hooks:
  webhook:
    default:
      urls:
        - http://alertmanager:9093/api/v1/alerts
        - http://webhook-service:8080/skywalking-alert

# Prometheus 告警 → SkyWalking（通过 API）
# SkyWalking 提供 API 输入外部告警

graph TD
    S["Service"] --> E["Exit Span<br/>数据库类型"]
    E --> OA["OAP 聚合"]
    OA --> SS["Slow SQL<br/>慢查询TOP N"]
    OA --> DT["DB Topology<br/>数据库依赖拓扑"]
    OA --> DM["Database Metrics<br/>延迟/吞吐量"]

UI → Database → Slow SQL
  展示：
  - SQL 语句（含参数）
  - 数据库类型（MySQL/PostgreSQL/Redis/MongoDB）
  - 执行次数
  - 平均延迟 / 最大延迟
  - 所属服务 / 实例
  - 最近执行时间

排序：按平均延迟 / 最大延迟 / 执行次数
过滤：按数据库类型 / 服务名 / 时间范围

拓扑图展示服务与数据库的依赖关系：
  user-service → PostgreSQL (users)
  user-service → Redis (cache)
  order-service → MySQL (orders)
  order-service → Elasticsearch (search)

连线信息：
  - 调用量 (cpm)
  - 平均延迟 (ms)
  - SLA (%)

节点颜色：健康状态
  - 绿: SLA >= 99%
  - 黄: SLA 90-99%
  - 红: SLA < 90%

hooks:
  dingtalk:
    text-template: |
      SkyWalking 告警通知
      告警规则: {ruleName}
      服务名称: {scopeName}
      告警内容: {message}
      触发时间: {date}      
    dingtalk:
      webhooks:
        - url: https://oapi.dingtalk.com/robot/send?access_token=xxx
          secret: your-secret

hooks:
  wechat:
    text-template: |
      SkyWalking 监控告警
      ================
      规则: {ruleName}
      服务: {scopeName}
      详情: {message}
      时间: {date}      
    webhooks:
      - url: https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=xxx

hooks:
  slack:
    text-template: |
      *SkyWalking Alarm*
      *Rule:* {ruleName}
      *Service:* {scopeName}
      *Message:* {message}
      *Time:* {date}      
    webhooks:
      - url: https://hooks.slack.com/services/T0/B0/xxx

# 添加 Helm 仓库
helm repo add skywalking https://apache.github.io/skywalking-helm
helm repo update

# 部署 SkyWalking（使用 ES 存储）
helm upgrade --install skywalking skywalking/skywalking \
  --namespace skywalking \
  --create-namespace \
  --set oap.replicas=2 \
  --set oap.storageType=elasticsearch \
  --set elasticsearch.enabled=true \
  --values values.yaml

# OAP 配置
oap:
  name: skywalking-oap
  replicas: 2
  image:
    repository: apache/skywalking-oap-server
    tag: 9.7.0
  storageType: elasticsearch
  resources:
    requests:
      memory: 2Gi
      cpu: "1"
    limits:
      memory: 4Gi
      cpu: "2"
  env:
    SW_CORE_GRPC_PORT: 11800
    SW_CORE_REST_PORT: 12800
    SW_HEALTH_CHECKER: "true"
    SW_TELEMETRY: prometheus
    SW_TELEMETRY_PROMETHEUS_PORT: 1234

# UI 配置
ui:
  name: skywalking-ui
  image:
    repository: apache/skywalking-ui
    tag: 9.7.0
  replicas: 2
  ingress:
    enabled: true
    host: skywalking.example.com
  env:
    SW_OAP_ADDRESS: http://skywalking-oap:12800

# ES 存储
elasticsearch:
  enabled: true
  replicas: 3
  resources:
    requests:
      memory: 4Gi
  persistence:
    size: 200Gi

# Java Agent 配置
javaAgent:
  enabled: true
  config:
    agent.service_name: ${SW_AGENT_NAME:default}
    collector.backend_service: skywalking-oap:11800

OAP：
  - 至少 2 副本（HA）
  - storageType 选择（ES/MySQL/TiDB）
  - gRPC 端口 11800（Agent 连接）
  - REST 端口 12800（UI 连接）

UI：
  - SW_OAP_ADDRESS 指向 OAP REST 端口
  - Ingress 配置对外暴露

Agent 注入：
  - 通过 Init Container 或 Sidecar 注入
  - 配置 collector.backend_service

# agent/config/agent.config
# 缓冲队列大小
agent.span_limit_per_segment: ${SW_AGENT_SPAN_LIMIT:300}
agent.span_limit_per_segment_stack_depth: ${SW_AGENT_SPAN_LIMIT_PER_SEGMENT_STACK:200}

# 忽略指定端点减少采样
agent.ignore_suffix=${SW_AGENT_IGNORE_SUFFIX:.jpg,.jpeg,.js,.css,.png,.bmp,.ico,.gif}

每个 Span 平均大小: 2-5KB (含 Tags/Logs)
每日 Trace 量: 1 亿 Span
每日存储: 1亿 × 3KB × 1.5(ES 索引开销) × 2(副本) ≈ 900GB/天

优化：增加采样率、缩短保留期、降低副本数

OAP REST 端口: 12800
GraphQL 端点: http://oap:12800/graphql

# 按条件查询 Trace 列表
query queryBasicTraces($condition: TraceQueryCondition) {
    queryBasicTraces(condition: $condition) {
        traces {
            key: traceId
            endpointNames
            duration
            start
            isError
            traceIds
        }
        total
    }
}

# 变量
{
    "condition": {
        "queryDuration": {
            "start": "2026-05-26 09:00",
            "end": "2026-05-26 10:00"
        },
        "traceState": "ALL",
        "paging": {
            "pageNum": 1,
            "pageSize": 20,
            "needTotal": true
        },
        "queryOrder": "BY_DURATION",
        "tags": [
            {
                "key": "status_code",
                "value": "500"
            }
        ]
    }
}

query queryTrace($traceId: ID!) {
    queryTrace(traceId: $traceId) {
        spans {
            spanId
            parentSpanId
            startTime
            endTime
            operationName
            isError
            layer
            tags {
                key
                value
            }
            logs {
                time
                data {
                    key
                    value
                }
            }
            refs {
                traceId
                parentSegmentId
                parentSpanId
                type
            }
        }
    }
}

query queryTopology($duration: Duration!) {
    getAllServices(duration: $duration) {
        id
        name
    }
    getAllServiceDependencies(duration: $duration) {
        source {
            id
            name
        }
        target {
            id
            name
        }
        detail {
            cpm
            avgResponseTime
            sla
        }
    }
}

# 启用健康检查端点
health-checker:
  selector: ${SW_HEALTH_CHECKER:default}
  default:
    checkIntervalSeconds: ${SW_HEALTH_CHECK_INTERVAL_SECONDS:5}

# 健康检查端点
# http://oap:12800/health
# http://oap:12800/health/instance

# 响应示例
{
    "healthy": true,
    "modules": [
        {"name": "receiver-grpc", "state": "RUNNING"},
        {"name": "receiver-http", "state": "RUNNING"},
        {"name": "storage", "state": "RUNNING"},
        {"name": "alarm", "state": "RUNNING"}
    ],
    "liveness": "UP",
    "readiness": "READY"
}

# OAP 配置支持通过 Apollo 或 ZK 动态更新
configuration:
  selector: ${SW_CONFIGURATION:grpc}
  grpc:
    host: ${SW_CONFIG_GRPC_HOST:localhost}
    port: ${SW_CONFIG_GRPC_PORT:8080}
    period: ${SW_CONFIG_GRPC_PERIOD:60}

  # 动态可调整的配置
  # - trace 采样率
  # - 告警规则
  # - 存储 TTL
  # - Agent 端配置

Agent 连接故障转移：
  Agent → OAP-1 (primary)
          OAP-2 (backup)
          OAP-3 (backup)

  当 OAP-1 故障：
    Agent 自动切换到 OAP-2
    无数据丢失（Agent 缓冲队列）
    OAP-1 恢复后重新负载

OAP 集群故障转移：
  OAP 之间通过 Cluster Coordinator 监测
  节点故障 → 业务转移给健康节点
  数据从存储层读取（存储层 HA）

# docker-compose
version: '3'
services:
  banyandb:
    image: apache/skywalking-banyandb:latest
    ports:
      - "17912:17912"  # gRPC
      - "17913:17913"  # HTTP
    environment:
      - SW_STORAGE_BANYANDB_HOST=0.0.0.0
      - SW_STORAGE_BANYANDB_GRPC_PORT=17912
    volumes:
      - ./banyandb-data:/data

  oap:
    image: apache/skywalking-oap-server:latest
    environment:
      - SW_STORAGE=banyandb
      - SW_STORAGE_BANYANDB_HOST=banyandb
      - SW_STORAGE_BANYANDB_GRPC_PORT=17912

1. 更低存储成本
   - 专门为 Trace/Metrics 数据模型优化
   - 列式压缩，压缩比 10-15×

2. 更高写入性能
   - 流式写入（非索引写）
   - 无倒排索引构建开销

3. 更简单运维
   - 单二进制部署
   - 无 JVM 依赖
   - 自动 TTL

4. 原生 APM 数据模型
   - Trace/Metrics/Log 原生支持
   - 无需 mapping 定义

graph TD
    HN["Host/Node"] --> SR["SkyWalking Rover<br/>eBPF"]
    SR --> NM["Network Monitor<br/>TCP/UDP<br/>连接追踪"]
    SR --> PM["Process/Syscall Monitor<br/>Exec/Libc<br/>调用追踪"]
    NM --> OAP["OAP Server<br/>数据接收与分析"]
    PM --> OAP

// eBPF 程序示例：TCP 连接追踪
SEC("kprobe/tcp_connect")
int trace_connect(struct pt_regs *ctx) {
    struct sock *sk;
    sk = (struct sock *)PT_REGS_PARM1(ctx);

    // 记录连接事件
    u32 src_ip = BPF_CORE_READ(sk, __sk_common.skc_rcv_saddr);
    u32 dst_ip = BPF_CORE_READ(sk, __sk_common.skc_daddr);
    u16 dst_port = BPF_CORE_READ(sk, __sk_common.skc_dport);

    // 发送到用户态
    bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU,
                          &evt, sizeof(evt));
    return 0;
}

优势：
  - 不支持 Agent 的语言（如 C/C++/Rust）
  - 遗留系统（无法修改）
  - Sidecar 容器（Envoy 协议）

局限：
  - 只能追踪网络层（无应用层语义）
  - HTTP body 内容有限
  - 需 Linux 4.9+ 内核
  - root 权限运行

// 1. 定义拦截点注解
@AgentCacheDeclaration
public class MyFrameworkPlugin extends AbstractClassEnhancePluginDefine {

    @Override
    protected ClassMatch enhanceClass() {
        // 匹配目标类
        return NameMatch.byName("com.example.MyClient");
    }

    @Override
    public ConstructorInterceptPoint[] getConstructorsInterceptPoints() {
        // 构造函数拦截点
        return null;
    }

    @Override
    public InstanceMethodsInterceptPoint[] getInstanceMethodsInterceptPoints() {
        // 实例方法拦截点
        return new InstanceMethodsInterceptPoint[] {
            new InstanceMethodsInterceptPoint() {
                @Override
                public ElementMatcher<MethodDescription> getMethodsMatcher() {
                    return named("sendRequest");
                }

                @Override
                public String getMethodsInterceptor() {
                    return "com.example.MyInterceptor";
                }

                @Override
                public boolean isOverrideArgs() {
                    return false;
                }
            }
        };
    }
}

public class MyInterceptor implements InstanceMethodsAroundInterceptor {
    @Override
    public void beforeMethod(EnhancedInstance objInst, Method method,
                             Object[] allArguments, Class<?>[] argumentsTypes) {
        // 创建 Exit Span
        AbstractSpan span = ContextManager.createExitSpan(
            "MyClient/sendRequest",
            allArguments[0].toString() // target address
        );
        span.setComponent(ComponentsDefine.MY_FRAMEWORK);
    }

    @Override
    public Object afterMethod(EnhancedInstance objInst, Method method,
                              Object[] allArguments, Class<?>[] argumentsTypes,
                              Object ret) {
        ContextManager.stopSpan();
        return ret;
    }

    @Override
    public void handleMethodException(EnhancedInstance objInst, Method method,
                                       Object[] allArguments, Class<?>[] argumentsTypes,
                                       Throwable t) {
        ContextManager.activeSpan().log(t);
    }
}

<!-- pom.xml 插件打包 -->
<project>
    <dependencies>
        <dependency>
            <groupId>org.apache.skywalking</groupId>
            <artifactId>apm-agent-core</artifactId>
            <version>${skywalking.version}</version>
            <scope>provided</scope>
        </dependency>
    </dependencies>
</project>

<!-- 部署到 Agent 插件目录 -->
cp my-plugin.jar /path/to/agent/plugins/

agent/
├── activations/       # 激活插件
├── bootstrap-plugins/ # 启动插件（核心拦截点）
├── plugins/           # 标准插件（自动加载）
│   ├── apm-tomcat-xx.jar
│   ├── apm-spring-xx.jar
│   ├── apm-jdbc-xx.jar
│   └── my-plugin.jar   ← 自定义插件
└── config/
    └── agent.config