Kthena AI 推理平台面试题库

graph LR
    A["Volcano<br/>批调度引擎"] --> B["Kthena<br/>AI 推理平台"]
    B --> C["Control Plane<br/>CRD 控制器"]
    B --> D["Data Plane<br/>Router 路由网关"]
    C --> E["ModelBooster<br/>一键部署"]
    C --> F["ModelServing<br/>工作负载管理"]
    C --> G["AutoScaling<br/>弹性伸缩"]
    D --> H["Scheduler Pipeline<br/>智能调度"]
    D --> I["Load Balancer<br/>负载均衡"]

graph LR
    A["用户提交 CRD"] --> B["Control Plane"]
    B --> C["ModelBooster Controller"]
    B --> D["ModelServing Controller"]
    B --> E["ModelRoute Controller"]
    B --> F["AutoScaling Controller"]
    C --> G["创建 Pod/Service/ConfigMap"]
    D --> G
    G --> H["Data Plane"]
    H --> I["Kthena Router"]
    I --> J["Auth → Rate Limiting<br/>→ Fairness → Scheduling<br/>→ Load Balancing → Proxy"]
    J --> K["推理 Pod<br/>vLLM/SGLang/Triton"]

graph LR
    A["ModelBooster"] --> B["ModelServing"]
    A --> C["ModelServer"]
    A --> D["ModelRoute"]
    E["AutoScalingPolicy"] -.-> F["AutoScalingPolicyBinding"]
    F --> B
    B --> G["ServingGroup"]
    G --> H["Role<br/>Prefill/Decode/Aggregated"]
    H --> I["Entry Pod + Worker Pod"]

graph LR
    A["用户创建 ModelBooster CRD"] --> B["Initialized<br/>控制器初始化子资源"]
    B --> C["Active<br/>所有子资源就绪"]
    B --> D["Failed<br/>创建失败"]
    D -.->|修复后重试| B

graph LR
    A["Entry Pod"] --> B["Runtime Agent<br/>Sidecar"]
    A --> C["LLM Engine<br/>vLLM/SGLang"]
    A --> D["Downloader<br/>Init Container"]
    D --> E["HuggingFace / S3<br/>/ OBS / PVC"]
    B --> F["指标采集<br/>LoRA 生命周期<br/>模型下载管理"]
    G["Worker Pod"] --> H["Runtime Agent<br/>Sidecar"]
    G --> I["LLM Engine<br/>推理引擎"]
    A -->|内部调度| G

graph LR
    A["推理请求"] --> B["Auth<br/>认证插件"]
    B --> C["Rate Limiting<br/>令牌桶限流"]
    C --> D["Fairness<br/>公平调度"]
    D --> E["Scheduling<br/>请求调度"]
    E --> F["Load Balancing<br/>负载均衡"]
    F --> G["Proxy<br/>转发至目标 Pod"]
    G --> H["推理引擎<br/>返回结果"]

graph LR
    A["候选 Pod 列表"] --> B["Filter 阶段"]
    B --> C["Least Requests<br/>过滤过载 Pod"]
    B --> D["LoRA Affinity<br/>过滤无适配器 Pod"]
    C --> E["Score 阶段"]
    D --> E
    E --> F["KV Cache Aware<br/>KV 缓存感知评分"]
    E --> G["Least Latency<br/>最低延迟评分"]
    E --> H["Prefix Cache<br/>前缀缓存评分"]
    E --> I["GPU Cache<br/>GPU 缓存评分"]
    F --> J["加权合并得分<br/>选择最优 Pod"]
    G --> J
    H --> J
    I --> J

graph LR
    A["用户请求<br/>Prompt + max_tokens"] --> B["Prefill Role<br/>计算密集"]
    B --> C["处理 Prompt tokens<br/>生成初始 KV Cache"]
    C --> D["KV Cache 传输<br/>LMCache/MoonCake/NIXL"]
    D --> E["Decode Role<br/>显存密集"]
    E --> F["逐 Token 解码<br/>返回最终响应"]

graph LR
    A["ModelServing"] --> B["ServingGroup"]
    B --> C["Role: Prefill<br/>副本数: 2"]
    B --> D["Role: Decode<br/>副本数: 4"]
    B --> E["Role: Aggregated<br/>副本数: 3"]
    C --> F["Entry Pod + Worker Pod"]
    D --> G["Entry Pod + Worker Pod × N"]
    E --> H["Entry Pod（兼任 Worker）"]

graph LR
    A["ModelBooster 创建"] --> B["创建 Volcano PodGroup<br/>minMember = 总 Pod 数"]
    B --> C["创建 Prefill Role Pods<br/>subGroupPolicy"]
    B --> D["创建 Decode Role Pods<br/>subGroupPolicy"]
    C --> E{"Volcano 调度器<br/>检查资源是否满足全部 Pod"}
    D --> E
    E -->|资源充足| F["全部 Pod 同时启动<br/>PodGroup Running"]
    E -->|资源不足| G["全部 Pod 等待<br/>PodGroup Pending"]

apiVersion: serving.kthena.volcano.sh/v1alpha1
kind: ModelBooster
metadata:
  name: llama3-70b
spec:
  scheduling:
    gangScheduling:
      enabled: true
      minAvailable: 6
    subGroupPolicy:
      - role: prefill
        minAvailable: 2
      - role: decode
        minAvailable: 4

graph LR
    A["AutoScalingPolicy"] --> B["Homogeneous<br/>同构伸缩"]
    A --> C["Heterogeneous<br/>异构伸缩"]
    B --> D["单一实例类型<br/>如全部 A100"]
    D --> E["Stable Mode<br/>平稳扩缩"]
    D --> F["Panic Mode<br/>突发扩容"]
    C --> G["多实例类型混合<br/>如 H100 + A100"]
    G --> H["optimizerConfiguration<br/>成本最优调度"]

graph LR
    A["AutoScalingPolicyBinding"] --> B["optimizerConfiguration<br/>成本优化配置"]
    B --> C["实例类型优先级<br/>A100 > H100（成本维度）"]
    B --> D["伸缩策略<br/>先扩低成本实例"]
    C --> E["ModelServing<br/>创建混合副本"]
    D --> E
    E --> F["A100 Pod × N<br/>成本优先"]
    E --> G["H100 Pod × M<br/>性能优先"]

apiVersion: autoscaling.kthena.volcano.sh/v1alpha1
kind: AutoScalingPolicyBinding
metadata:
  name: llama3-hetero
spec:
  modelServingRef:
    name: llama3-70b
  policyName: llama3-policy
  optimizerConfiguration:
    strategy: cost-driven
    instanceTypes:
      - type: nvidia-a100-80g
        priority: 1
        maxReplicas: 8
      - type: nvidia-h100
        priority: 2
        maxReplicas: 4

graph LR
    A["推理请求"] --> B["Kthena Router"]
    B --> C["ModelRoute<br/>路由规则匹配"]
    C -->|"90% 流量"| D["ModelServing v1<br/>稳定版本"]
    C -->|"10% 流量"| E["ModelServing v2<br/>Canary 版本"]
    E --> F{"指标验证<br/>延迟/错误率"}
    F -->|通过| G["逐步提升 v2 流量比例"]
    F -->|不通过| H["回退至 v1"]

apiVersion: serving.kthena.volcano.sh/v1alpha1
kind: ModelRoute
metadata:
  name: llama3-route
spec:
  routes:
    - modelServingRef: llama3-v1
      weight: 90
    - modelServingRef: llama3-v2-canary
      weight: 10
  failover:
    enabled: true
    targetRef: llama3-v1-fallback

graph LR
    A["推理请求<br/>含 Prompt + max_tokens"] --> B["Rate Limiting 插件"]
    B --> C["令牌桶算法<br/>按 Tenant/Model/Key 匹配规则"]
    C -->|"配额充足"| D["放行请求"]
    C -->|"配额不足"| E["返回 429 Too Many Requests"]
    B --> F["Token 计量<br/>Prompt tokens + Completion tokens"]
    F --> G["实时扣减配额"]

graph LR
    A["用户创建/更新<br/>LoRA 适配器 CRD"] --> B["Runtime Agent<br/>检测到适配器变更"]
    B --> C["下载适配器权重<br/>从 Storage 加载"]
    C --> D["热加载至推理引擎<br/>vLLM/SGLang"]
    D --> E["上报适配器状态<br/>至 Router"]
    E --> F["LoRA Affinity 插件<br/>更新路由表"]
    F --> G["后续请求自动路由<br/>至已加载的 Pod"]

graph LR
    A["推理引擎选型"] --> B{"工作负载类型"}
    B -->|LLM 高并发| C["vLLM<br/>PagedAttention + Continuous Batching"]
    B -->|LLM 低延迟| D["SGLang<br/>RadixAttention + KV Cache 复用"]
    B -->|多框架混合| E["Triton<br/>TensorRT + PyTorch + ONNX"]
    B -->|PyTorch 生态| F["TorchServe<br/>原生 PyTorch 部署"]

graph LR
    A["Pod 启动"] --> B["Downloader<br/>Init Container"]
    B --> C{"模型来源"}
    C -->|"HuggingFace"| D["HF Hub API<br/>下载权重"]
    C -->|"S3"| E["S3 SDK<br/>下载权重"]
    C -->|"OBS"| F["华为 OBS SDK<br/>下载权重"]
    C -->|"PVC"| G["从 PVC 挂载<br/>直接读取"]
    D --> H["权重写入共享存储"]
    E --> H
    F --> H
    G --> H
    H --> I["Runtime Agent<br/>校验权重完整性"]
    I --> J["启动推理引擎"]

graph LR
    A["HyperNode CRD<br/>定义网络拓扑"] --> B["节点分组<br/>Rack / AZ / Region"]
    B --> C["Volcano 调度器<br/>感知拓扑标签"]
    C --> D["Prefill Pod<br/>调度至 Node A"]
    C --> E["Decode Pod<br/>调度至同 Rack Node B"]
    D -->|"低延迟网络<br/>KV Cache 传输"| E

graph LR
    A["ModelBooster CRD"] --> B["Controller"]
    B --> C["GPU Role<br/>nvidia.com/gpu"]
    B --> D["NPU Role<br/>huawei.com/ascend"]
    C --> E["vLLM + CUDA"]
    D --> F["MindSpore / PyTorch<br/>+ HCCL"]
    E --> G["推理服务<br/>统一暴露"]
    F --> G

spec:
  servingGroups:
    - roles:
        - name: gpu-prefill
          resource:
            nvidia.com/gpu: 4
          replicas: 2
        - name: npu-decode
          resource:
            huawei.com/ascend: 8
          replicas: 3

graph LR
    A["Prefill Role<br/>生成 KV Cache"] --> B["KV Cache 连接器<br/>LMCache/MoonCake/NIXL"]
    B --> C["传输至<br/>Decode Role"]
    C --> D["Decode Role<br/>加载 KV Cache<br/>继续解码"]
    A --> E["KV Cache 复用<br/>相同 Prefix 的请求<br/>可跳过 Prefill"]

graph LR
    A["ModelServer CRD"] --> B["Controller"]
    B --> C["Kubernetes Service<br/>ClusterIP / NodePort / LoadBalancer"]
    B --> D["Ingress<br/>域名路由 + TLS"]
    B --> E["ModelRoute<br/>流量路由规则"]
    C --> F["Kthena Router<br/>Service 后端"]
    F --> G["推理 Pod<br/>实际处理请求"]

graph LR
    A["Router 定期健康检查<br/>探测推理 Pod 状态"] --> B{"主实例健康"}
    B -->|"健康"| C["正常转发流量"]
    B -->|"异常"| D["标记主实例不可用"]
    D --> E["查询 failover 配置"]
    E --> F["切换至备用实例<br/>或同角色其他 Pod"]
    F --> G["持续探测主实例<br/>恢复后自动回切"]

graph LR
    A["Runtime Agent<br/>Sidecar"] --> B["采集推理引擎指标"]
    B --> C["Prometheus<br/>metrics 端点"]
    C --> D["Grafana Dashboard<br/>推理监控面板"]
    C --> E["告警规则<br/>延迟/错误率/资源"]

apiVersion: serving.kthena.volcano.sh/v1alpha1
kind: ModelBooster
metadata:
  name: llama3-8b
spec:
  model:
    name: meta-llama/Meta-Llama-3-8B-Instruct
    source:
      type: huggingface
      repoID: meta-llama/Meta-Llama-3-8B-Instruct
  engine:
    type: vllm
    version: 0.6.0
    args:
      - --max-model-len=8192
      - --gpu-memory-utilization=0.9
  resources:
    nvidia.com/gpu: 1
    memory: 16Gi
  replicas: 2
  scheduling:
    gangScheduling:
      enabled: true
  service:
    type: ClusterIP
    port: 8000

apiVersion: serving.kthena.volcano.sh/v1alpha1
kind: ModelServing
metadata:
  name: llama3-70b-pd
spec:
  model:
    name: meta-llama/Meta-Llama-3-70B-Instruct
    source:
      type: s3
      endpoint: s3.amazonaws.com
      bucket: my-models
  servingGroups:
    - name: pd-group
      roles:
        - name: prefill
          engine:
            type: vllm
            args:
              - --role=prefill
              - --kv-connector=mooncake
          resources:
            nvidia.com/gpu: 4
          replicas: 2
        - name: decode
          engine:
            type: vllm
            args:
              - --role=decode
              - --kv-connector=mooncake
          resources:
            nvidia.com/gpu: 2
          replicas: 4
  scheduling:
    gangScheduling:
      enabled: true
    topologyAware:
      enabled: true
      topologyKey: topology.kubernetes.io/zone

graph LR
    A{"选择 Role 类型"} -->|"模型 < 30B<br/>延迟要求一般"| B["Aggregated<br/>Prefill + Decode 合并"]
    A -->|"模型 ≥ 30B<br/>高并发场景"| C["PD 分离<br/>Prefill + Decode 独立"]
    B --> D["优点：简单<br/>缺点：资源利用率受限"]
    C --> E["优点：独立扩缩容<br/>缺点：架构复杂度高"]

graph LR
    A["Team A<br/>ModelBooster"] --> B["Queue: team-a<br/>配额: 16 GPU"]
    C["Team B<br/>ModelBooster"] --> D["Queue: team-b<br/>配额: 8 GPU"]
    B --> E["Volcano 调度器<br/>按 Queue 配额分配"]
    D --> E
    E --> F["节点资源池<br/>公平分配"]

apiVersion: scheduling.volcano.sh/v1beta1
kind: Queue
metadata:
  name: inference-team-a
spec:
  weight: 2
  capability:
    nvidia.com/gpu: 16
    cpu: "64"
    memory: 256Gi

graph LR
    A["Runtime Agent<br/>Sidecar"] --> B["模型下载协调<br/>管理 Downloader 状态"]
    A --> C["指标采集<br/>Prometheus metrics"]
    A --> D["LoRA 生命周期<br/>加载/卸载适配器"]
    A --> E["健康上报<br/>就绪探针"]
    A --> F["引擎适配<br/>vLLM/SGLang 统一接口"]
    C --> G["Prometheus<br/>Server"]
    E --> H["Kthena Controller<br/>状态同步"]
    D --> I["Router<br/>适配器路由表"]

graph LR
    A["Pod 启动失败"] --> B{"Pod 状态"}
    B -->|"Pending"| C["检查资源<br/>GPU/Memory 是否充足"]
    B -->|"Pending"| D["检查 Gang Scheduling<br/>PodGroup 是否 Running"]
    B -->|"ImagePullBackOff"| E["检查推理引擎镜像<br/>是否可拉取"]
    B -->|"Init:Error"| F["检查 Downloader<br/>模型下载日志"]
    B -->|"CrashLoopBackOff"| G["检查推理引擎<br/>启动日志"]
    C --> H["kubectl describe pod<br/>查看事件"]
    D --> I["kubectl get podgroup<br/>查看调度状态"]
    F --> J["kubectl logs pod<br/>-c downloader"]
    G --> K["kubectl logs pod<br/>-c engine"]

graph LR
    A["客户端请求"] --> B["Load Balancer<br/>多 Router 实例"]
    B --> C["Kthena Router 1"]
    B --> D["Kthena Router 2"]
    C --> E["ModelServing<br/>多副本部署"]
    D --> E
    E --> F["Prefill Role × N"]
    E --> G["Decode Role × M"]
    F --> H["跨 AZ 部署<br/>拓扑感知调度"]
    G --> H