Ceph / Rook 面试题

应用层        RBD (块设备)   CephFS (文件系统)   RGW (对象存储)
              ──────────────────────────────────────────
内核/用户态   librbd          libcephfs         librgw
              ──────────────────────────────────────────
底层存储      RADOS (librados)
              ──────────────────────────────────────────
              MON 集群 (Paxos)  +  OSD 集群 (CRUSH)

rule replicated_rule {
    id 0
    type replicated
    min_size 1
    max_size 10
    step take default
    step chooseleaf firstn 0 type host  // 选择不同 Host 上的 OSD
    step emit
}

Pool (逻辑存储池)
  └── PG (Placement Group)
        └── Object (实际存储对象)

配置 (bluestore)                         rocksdb WAL     rocksdb DB     对象数据
────────────────────────────────────────────────────────────────────────────
仅主盘                                   主盘             主盘             主盘
主盘 + DB/WAL 分区                       主盘独立分区      主盘独立分区      主盘
主盘 + 独立 SSD                           独立 SSD         独立 SSD         主盘
主盘 + 独立 WAL (NVMe) + 独立 DB (SSD)     独立 NVMe        独立 SSD         主盘

[osd]
bluestore_cache_size_hdd = 4GB       # HDD OSD 缓存
bluestore_cache_size_ssd = 4GB       # SSD OSD 缓存
bluestore_cache_trim_interval = 0.05 # Trim 间隔
bluestore_throttle_cost_per_io = 1.5 # I/O 节流系数

graph TD
    subgraph RookOperator["Rook Operator"]
        CC["Cluster\nController"]
        BC["Block\nController"]
        FC["Filesys\nController"]
        OC["Object\nController"]
    end
    subgraph RookAgent["Rook Agent - DaemonSet"]
        Agent["每节点运行，管理 OSD / 设备发现 / 配置注入"]
    end
    subgraph CephPods["Ceph 守护进程 Pod"]
        MON["MON Pod"]
        OSD["OSD Pod"]
        MDS["MDS Pod"]
        RGW["RGW Pod"]
        MGR["MGR Pod"]
    end
    subgraph CSI["CSI 驱动 Pod"]
        RBD["RBD CSI Plugin"]
        CephFS["CephFS CSI Plugin"]
    end
    RookOperator --> RookAgent
    RookAgent --> CephPods
    CephPods --> CSI

# 1. 克隆 Rook 仓库，部署 Operator
git clone --single-branch --branch v1.14.0 https://github.com/rook/rook.git
kubectl create -f rook/deploy/examples/crds.yaml     # 创建 CRD
kubectl create -f rook/deploy/examples/common.yaml   # 创建 RBAC / ServiceAccount
kubectl create -f rook/deploy/examples/operator.yaml # 部署 Operator

# 2. 创建 CephCluster CR
kubectl apply -f cluster.yaml

apiVersion: ceph.rook.io/v1
kind: CephCluster
metadata:
  name: rook-ceph
  namespace: rook-ceph
spec:
  cephVersion:
    image: quay.io/ceph/ceph:v18.2.0
  dataDirHostPath: /var/lib/rook
  mon:
    count: 3                        # MON 数量，建议奇数 ≥ 3
    allowMultiplePerNode: false     # 禁止同一节点多 MON
  mgr:
    count: 2
    modules:
      - name: pg_autoscaler
        enabled: true
  dashboard:
    enabled: true
    ssl: true
  network:
    provider: host                  # host 或 multus
    selectors:
      public: public-network
      cluster: cluster-network
  storage:
    useAllNodes: true               # 自动发现节点
    useAllDevices: true             # 自动发现空白设备
    deviceFilter: "^sd[b-d]"       # 设备名过滤
    config:
      osdsPerDevice: "1"            # 每设备 OSD 数（NVMe 可设为 > 1）
    nodes:
      - name: node-1
        devices:
          - name: /dev/nvme0n1
            config:
              osdsPerDevice: "4"
  resources:
    osd:
      limits:
        cpu: "4"
        memory: "16Gi"
      requests:
        cpu: "2"
        memory: "8Gi"

kubectl -n rook-ceph get cephcluster
kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph status
kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph osd tree

apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
  name: replicapool
  namespace: rook-ceph
spec:
  failureDomain: host              # 故障域级别
  replicated:
    size: 3                         # 副本数
    requireSafeReplicaSize: true    # 要求可用副本才允许写入
  enableRBDStats: true              # 开启 RBD 性能统计
  parameters:
    pg_num: "128"
    pgp_num: "128"
    compression_mode: aggressive
    target_size_ratio: 0.3          # PG Autoscaler 权重

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: rook-ceph-block
provisioner: rook-ceph.rbd.csi.ceph.com
parameters:
  clusterID: rook-ceph
  pool: replicapool
  imageFormat: "2"
  imageFeatures: layering,exclusive-lock,object-map,fast-diff,deep-flatten
  csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
  csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
  csi.storage.k8s.io/fstype: ext4
allowVolumeExpansion: true
reclaimPolicy: Delete
mountOptions:
  - discard                     # 启用 TRIM / UNMAP

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: mysql-data
spec:
  storageClassName: rook-ceph-block
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 100Gi
apiVersion: v1
kind: Pod
metadata:
  name: mysql
spec:
  containers:
    - name: mysql
      image: mysql:8.0
      volumeMounts:
        - name: data
          mountPath: /var/lib/mysql
  volumes:
    - name: data
      persistentVolumeClaim:
        claimName: mysql-data

kubectl patch pvc mysql-data -p '{"spec":{"resources":{"requests":{"storage":"200Gi"}}}}'

apiVersion: snapshot.storage.k8s.io/v1
kind: VolumeSnapshotClass
metadata:
  name: csi-rbdplugin-snapclass
driver: rook-ceph.rbd.csi.ceph.com
deletionPolicy: Delete
apiVersion: snapshot.storage.k8s.io/v1
kind: VolumeSnapshot
metadata:
  name: mysql-snapshot
spec:
  volumeSnapshotClassName: csi-rbdplugin-snapclass
  source:
    persistentVolumeClaimName: mysql-data
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: mysql-restore
spec:
  storageClassName: rook-ceph-block
  dataSource:
    name: mysql-snapshot
    kind: VolumeSnapshot
    apiGroup: snapshot.storage.k8s.io
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 100Gi

apiVersion: ceph.rook.io/v1
kind: CephFilesystem
metadata:
  name: myfs
  namespace: rook-ceph
spec:
  metadataPool:
    replicated:
      size: 3
    parameters:
      pg_num: "128"
      compression_mode: "none"      # 元数据池不建议压缩
  dataPools:
    - name: replicated
      failureDomain: host
      replicated:
        size: 3
      parameters:
        pg_num: "256"
        compression_mode: aggressive
    - name: ec-data                # 纠删码数据池
      failureDomain: host
      erasureCoded:
        dataChunks: 6
        codingChunks: 2            # 6+2 EC，1.33x 冗余率
  preserveFilesystemOnDelete: false
  metadataServer:
    activeCount: 1
    activeStandby: true             # 启用 standby MDS
    resources:
      limits:
        cpu: "4"
        memory: "8Gi"
      requests:
        cpu: "2"
        memory: "4Gi"

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: rook-cephfs
provisioner: rook-ceph.cephfs.csi.ceph.com
parameters:
  clusterID: rook-ceph
  fsName: myfs
  pool: myfs-replicated
  csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
  csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
allowVolumeExpansion: true
reclaimPolicy: Delete

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: shared-data
spec:
  storageClassName: rook-cephfs
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 500Gi
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-shared
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nginx-shared
  template:
    spec:
      containers:
        - name: nginx
          image: nginx:1.25
          volumeMounts:
            - name: data
              mountPath: /usr/share/nginx/html
      volumes:
        - name: data
          persistentVolumeClaim:
            claimName: shared-data

apiVersion: ceph.rook.io/v1
kind: CephFilesystemSubVolumeGroup
metadata:
  name: tenant-a-group
  namespace: rook-ceph
spec:
  filesystemName: myfs
# StorageClass 通过 subvolumegroup 参数指向租户组
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: rook-cephfs-tenant-a
provisioner: rook-ceph.cephfs.csi.ceph.com
parameters:
  clusterID: rook-ceph
  fsName: myfs
  subvolumegroup: tenant-a-group

apiVersion: ceph.rook.io/v1
kind: CephObjectStore
metadata:
  name: my-store
  namespace: rook-ceph
spec:
  metadataPool:
    replicated:
      size: 3
  dataPool:
    replicated:
      size: 3
    erasureCoded:
      dataChunks: 4
      codingChunks: 2
  preservePoolsOnDelete: true
  gateway:
    type: s3
    port: 80
    securePort: 443
    instances: 3
    sslCertificateRef: rgw-tls-cert   # TLS Secret 引用
    resources:
      limits:
        cpu: "4"
        memory: "8Gi"
      requests:
        cpu: "2"
        memory: "4Gi"
  healthCheck:
    bucket:
      disabled: false
      interval: 60s

apiVersion: ceph.rook.io/v1
kind: CephObjectStoreUser
metadata:
  name: my-user
  namespace: rook-ceph
spec:
  store: my-store
  displayName: "S3 ReadWrite User"
  quotas:
    maxBuckets: 100
    maxSize: 1024Gi
    maxObjects: 1000000
  capabilities:
    - user: "*"
    - bucket: "*"
    - usage: read
    - metadata: read

# 获取 Access Key
kubectl -n rook-ceph get secret rook-ceph-object-user-my-store-my-user \
  -o jsonpath='{.data.AccessKey}' | base64 --decode

# 获取 Secret Key
kubectl -n rook-ceph get secret rook-ceph-object-user-my-store-my-user \
  -o jsonpath='{.data.SecretKey}' | base64 --decode

# 获取 RGW Service Endpoint
kubectl -n rook-ceph get svc rook-ceph-rgw-my-store -o jsonpath='{.spec.clusterIP}'

s3cmd --access_key=<AccessKey> --secret_key=<SecretKey> \
  --host=<RGW-Endpoint> --host-bucket="" \
  --no-ssl mb s3://my-bucket

apiVersion: v1
kind: Service
metadata:
  name: rgw-external-lb
  namespace: rook-ceph
spec:
  type: LoadBalancer
  selector:
    app: rook-ceph-rgw
    rgw: my-store
  ports:
    - name: http
      port: 80
      targetPort: 8080
    - name: https
      port: 443
      targetPort: 8443

apiVersion: ceph.rook.io/v1
kind: CephBlockPoolRadosNamespace
metadata:
  name: tenant-x-ns
  namespace: rook-ceph
spec:
  blockPoolName: replicapool
# StorageClass 指向特定 Namespace
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: rook-ceph-block-tenant-x
provisioner: rook-ceph.rbd.csi.ceph.com
parameters:
  clusterID: rook-ceph
  pool: replicapool
  radosNamespace: tenant-x-ns

apiVersion: ceph.rook.io/v1
kind: CephRBDMirror
metadata:
  name: rbd-mirror
  namespace: rook-ceph
spec:
  count: 1
  resources:
    limits:
      cpu: "2"
      memory: "4Gi"
  peers:
    secretNames:
      - cluster-b-peer-secret
  pool: replicapool
  mode: pool                  # pool 或 image 级别
  snapshotSchedules:
    - interval: 15m
      startTime: "00:00:00"

# 查看 Trash 中的镜像
rbd trash ls -p replicapool
# 恢复已删除镜像
rbd trash restore -p replicapool <image-id>

graph TD
    subgraph K8sMaster["Kubernetes Master"]
        Prov["CSI Provisioner"]
        Snap["CSI Snapshotter / Resizer"]
        Prov --> Ctrl
        Snap --> Ctrl
        Ctrl["CSI Controller Plugin\nrook-ceph.rbd.csi.ceph.com /\nrook-ceph.cephfs.csi.ceph.com"]
    end
    subgraph K8sNode["Kubernetes Node"]
        Node["CSI Node Plugin\nDaemonSet: csi-rbdplugin /\ncsi-cephfsplugin"]
        Stage["NodeStage\nVolume"]
        Publish["NodePublish\nVolume"]
    end
    Ctrl --> Node

# 外部 K8s 集群 StorageClass
parameters:
  clusterID: rook-ceph-external
  pool: replicapool
  csi.storage.k8s.io/provisioner-secret-namespace: external-ns

apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
  name: nvme-fast-pool
  namespace: rook-ceph
spec:
  failureDomain: host
  deviceClass: nvme
  replicated:
    size: 3
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
  name: hdd-archive-pool
  namespace: rook-ceph
spec:
  failureDomain: host
  deviceClass: hdd
  replicated:
    size: 2

apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
  name: ssd-primary-pool
  namespace: rook-ceph
spec:
  failureDomain: rack
  deviceClass: ssd
  replicated:
    size: 3
  crushRoot: ssd-root          # 限定 CRUSH tree 根节点

# NVMe 高性能 StorageClass
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: rook-nvme-block
provisioner: rook-ceph.rbd.csi.ceph.com
parameters:
  pool: nvme-fast-pool

# HDD 归档 StorageClass
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: rook-hdd-block
provisioner: rook-ceph.rbd.csi.ceph.com
parameters:
  pool: hdd-archive-pool

apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
  name: replicapool
  namespace: rook-ceph
spec:
  parameters:
    pg_autoscale_mode: "on"
    target_size_ratio: 0.3     # Pool 占集群总容量比例，用于计算 PG

# 查看 Pool 的 PG Autoscaler 状态
ceph osd pool autoscale-status

# 手动为 Pool 设置 target ratio
ceph osd pool set replicapool target_size_ratio 0.5

# 查看 OSD 树
kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph osd tree

# 查看 OSD 状态
kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph osd df

# 1. 标记 OSD 为 out（集群自动迁移数据）
kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph osd out osd.<N>

# 2. 等待数据迁移完成
kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph -w

# 3. 停止 OSD 进程
kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph osd down osd.<N>

# 4. 删除 OSD（从 CRUSH Map 和 Auth 中移除）
kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph osd purge osd.<N>

# 5. 清理节点上的设备
# 如使用 Rook 管理的 OSD，通过删除对应 deployment 触发清理 Job
kubectl -n rook-ceph delete deployment rook-ceph-osd-<N>

# 6. 物理移除磁盘

# 1. 执行逐出（evict），替换 out
ceph osd destroy osd.<N> --force
# 2. 物理更换磁盘
# 3. Rook Agent 检测到新盘，自动创建新 OSD
# 4. 集群自动再平衡

# 调整 OSD 权重（根据容量差异微调）
ceph osd crush reweight osd.<N> <weight>

# 批量平衡（upmap 优化器）
ceph balancer mode upmap
ceph balancer on

apiVersion: ceph.rook.io/v1
kind: CephCluster
spec:
  mon:
    count: 3
    allowMultiplePerNode: false
    volumeClaimTemplate:
      spec:
        storageClassName: local-storage    # MON 数据使用本地 PV
        resources:
          requests:
            storage: 20Gi

# 1. 检查 MON 状态
kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph mon stat

# 2. 如 MON 数据完整，重启失败 MON Pod
kubectl -n rook-ceph delete pod rook-ceph-mon-a-xxxx

# 3. 如 MON 数据损坏，从健康 MON 提取 monmap 注入新 MON
kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph-monstore-tool \
  /var/lib/rook/mon-a get monmap -- --out /tmp/monmap

# 4. 将 MON 从集群中移除并重建
ceph mon remove a

apiVersion: ceph.rook.io/v1
kind: CephFilesystem
metadata:
  name: myfs
  namespace: rook-ceph
spec:
  metadataServer:
    activeCount: 2
    activeStandby: true
    resources:
      limits:
        cpu: "8"
        memory: "16Gi"
      requests:
        cpu: "4"
        memory: "8Gi"

[mds]
mds_cache_memory_limit = 16GB          # MDS 内存缓存上限
mds_cache_reservation = 0.05           # 缓存保留比例
mds_recall_state_timeout = 60          # 客户端状态回收超时
mds_session_timeout = 60               # 客户端会话超时
mds_bal_interval = 10                  # MDS 负载均衡间隔（秒）
mds_log_max_segments = 128             # MDS 日志段数量
mds_max_file_recover = 32              # 恢复期间最大打开文件数

# MDS 性能计数器
ceph daemon mds.myfs-a perf dump

# 客户端会话
ceph tell mds.myfs:a client ls

# 查看子树分区分布
ceph fs status myfs

graph TD
    Req["HTTP/HTTPS Request"]
    LB["LoadBalancer\nEnvoy / HAProxy / Nginx"]
    RGW1["RGW Pod"]
    RGW2["RGW Pod"]
    RGW3["RGW Pod"]
    RADOS["RADOS"]
    Req --> LB
    LB --> RGW1
    LB --> RGW2
    LB --> RGW3
    RGW1 --> RADOS
    RGW2 --> RADOS
    RGW3 --> RADOS

apiVersion: ceph.rook.io/v1
kind: CephObjectStore
metadata:
  name: cluster-a-store
  namespace: rook-ceph
spec:
  gateway:
    instances: 5
    port: 80
    securePort: 443
  zone:
    name: zone-a
  healthCheck:
    bucket:
      interval: 30s

[client.rgw]
rgw_frontends = "beast port=80"
rgw_thread_pool_size = 512
rgw_max_concurrent_requests = 1024
rgw_cache_enabled = true
rgw_cache_lru_size = 100000
rgw_gc_max_objs = 32
rgw_gc_processor_period = 3600

# Beast 绑核（减少上下文切换）
rgw_frontends = "beast port=80 num_threads=4"

# 限制请求体大小
rgw_max_put_size = 5GiB
rgw_max_get_size = 10GiB

apiVersion: ceph.rook.io/v1
kind: CephCluster
spec:
  dashboard:
    enabled: true
    ssl: true
    urlPrefix: /
    port: 8443
    sslCertificateSecretName: dashboard-cert  # 自定义 TLS

# 首次部署后获取 admin 密码
kubectl -n rook-ceph get secret rook-ceph-dashboard-password \
  -o jsonpath="{['data']['password']}" | base64 --decode

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: rook-ceph-dashboard
  namespace: rook-ceph
  annotations:
    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
spec:
  rules:
    - host: ceph-dashboard.example.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: rook-ceph-mgr-dashboard
                port:
                  number: 8443
  tls:
    - hosts:
        - ceph-dashboard.example.com
      secretName: dashboard-tls

# 创建只读用户
kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- \
  ceph dashboard ac-user-create readonly-user <password> read-only

# 授予特定权限
ceph dashboard ac-user-set-roles admin-user administrator

graph LR
    subgraph ClusterA["集群 A"]
        RBD1["RBD Image<br/>(Primary)<br/>Snapshot Schedule"]
        Mirror1["CephRBD Mirror Daemon"]
        RBD1 --> Mirror1
    end
    subgraph ClusterB["集群 B"]
        RBD2["RBD Image<br/>(Secondary)<br/>Snapshot Schedule"]
        Mirror2["CephRBD Mirror Daemon"]
        RBD2 --> Mirror2
    end
    RBD1 -->|"Journal 同步"| RBD2

apiVersion: ceph.rook.io/v1
kind: CephRBDMirror
metadata:
  name: rbd-mirror
  namespace: rook-ceph
spec:
  count: 2
  pool: replicapool
  mode: image
  peers:
    secretNames:
      - cluster-b-peer-secret

apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
  name: replicapool
  namespace: rook-ceph
spec:
  mirroring:
    enabled: true
    mode: image
    snapshotSchedules:
      - interval: 5m
        startTime: "00:00:00"
    maxSnapshots: 5

graph TD
    Realm["Realm: global"]
    ZoneA["Zone A<br/>(Primary)<br/>Site: beijing"]
    ZoneB["Zone B<br/>(Secondary)<br/>Site: shanghai"]
    Realm --> ZoneA
    Realm --> ZoneB
    ZoneA <-->|"Sync"| ZoneB

apiVersion: ceph.rook.io/v1
kind: CephObjectRealm
metadata:
  name: global-realm
  namespace: rook-ceph

apiVersion: ceph.rook.io/v1
kind: CephObjectZoneGroup
metadata:
  name: default-group
  namespace: rook-ceph
spec:
  realm: global-realm

apiVersion: ceph.rook.io/v1
kind: CephObjectZone
metadata:
  name: zone-a
  namespace: rook-ceph
spec:
  zoneGroup: default-group
  metadataPool:
    replicated:
      size: 3
  dataPool:
    replicated:
      size: 3

# 1. 检查当前集群状态
kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph versions
kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph status

# 2. 更新 CephCluster 镜像
kubectl -n rook-ceph patch cephcluster rook-ceph --type merge \
  -p '{"spec":{"cephVersion":{"image":"quay.io/ceph/ceph:v18.2.1","allowUnsupported":false}}}'

# 3. 观察升级进度
kubectl -n rook-ceph logs -f deploy/rook-ceph-operator
kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph versions

v15.2.x (Octopus) → v16.2.x (Pacific) → v17.2.x (Quincy) → v18.2.x (Reef)

# 1. 从 Ceph 集群导出连接信息
kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- \
  python3 /etc/ceph/create-external-cluster-resources.py \
  --rbd-data-pool-name replicapool \
  --cephfs-filesystem-name myfs \
  --namespace rook-ceph-external

# 2. 在外部 K8s 集群上创建连接 Secret
kubectl create -f external-cluster-secret.yaml

# 3. 外部 K8s 集群 StorageClass 指向外部 Ceph
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: rook-ceph-block-external
provisioner: rook-ceph.rbd.csi.ceph.com
parameters:
  clusterID: rook-ceph-external
  pool: replicapool

[global]
ms_type = async+posix                   # 异步消息层
ms_async_op_threads = 4                 # OSD 工作线程
ms_bind_port_min = 6800
ms_bind_port_max = 7300

[osd]
osd_op_num_threads_per_shard = 2        # SSD 建议 2~4，HDD 建议 1
osd_op_num_shards = 8                   # OSD 分片数

# /etc/sysctl.d/90-ceph.conf
net.core.rmem_max = 134217728           # 128MB 接收缓冲区
net.core.wmem_max = 134217728           # 128MB 发送缓冲区
net.ipv4.tcp_rmem = 4096 87380 134217728
net.ipv4.tcp_wmem = 4096 65536 134217728
net.core.netdev_max_backlog = 300000

[osd]
# HDD 优化
bluestore_cache_size_hdd = 4GB
bluestore_cache_size_ssd = 4GB
bluestore_cache_kv_ratio = 0.6          # 60% 给 RocksDB 缓存，40% 给 onode 缓存
bluestore_cache_meta_ratio = 0.6        # 元数据缓存占比

# RocksDB 调优
bluestore_rocksdb_options = "compression=kLZ4Compression,max_write_buffer_number=8,min_write_buffer_number_to_merge=2,write_buffer_size=256MB,max_bytes_for_level_base=4GB,max_background_jobs=8"

[osd]
osd_memory_target = 8GB                 # Ceph 自动管理 OSD 内存上限
osd_memory_cache_min = 512MB            # 保留缓冲
bluestore_cache_autotune = true         # 自动调整 BlueStore 缓存

apiVersion: ceph.rook.io/v1
kind: CephCluster
spec:
  resources:
    osd:
      limits:
        cpu: "8"
        memory: "16Gi"
      requests:
        cpu: "2"
        memory: "4Gi"
    mon:
      limits:
        cpu: "2"
        memory: "4Gi"
    mgr:
      limits:
        cpu: "1"
        memory: "2Gi"

apiVersion: ceph.rook.io/v1
kind: CephCluster
spec:
  monitoring:
    enabled: true
    rulesNamespace: rook-ceph
  mgr:
    modules:
      - name: prometheus
        enabled: true

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: rook-ceph-mgr
  namespace: rook-ceph
spec:
  endpoints:
    - port: http-metrics
      interval: 15s
  selector:
    matchLabels:
      app: rook-ceph-mgr

groups:
  - name: ceph.rules
    rules:
      - alert: CephClusterDown
        expr: ceph_health_status > 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Ceph 集群不健康"

      - alert: CephOsdDown
        expr: ceph_osd_up == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Ceph OSD XQOPEN $labels.ceph_daemon XQCLOSE 已下线"

      - alert: CephPGsInactive
        expr: ceph_pg_inactive > 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "XQOPEN $value XQCLOSE 个 PG 处于 inactive 状态"

      - alert: CephPGsStuck
        expr: ceph_pg_stuck_inactive / ceph_pg_total > 0.01 or
              ceph_pg_stuck_unclean / ceph_pg_total > 0.01
        for: 10m
        labels:
          severity: critical

      - alert: CephClusterNearFull
        expr: ceph_cluster_total_used_bytes / ceph_cluster_total_bytes > 0.85
        labels:
          severity: critical

apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
  name: ec-pool
  namespace: rook-ceph
spec:
  failureDomain: host
  erasureCoded:
    dataChunks: 4
    codingChunks: 2
  parameters:
    crush-failure-domain: host

# buckets
host node-1 {
    id -3
    alg straw2
    hash 0
    item osd.0 weight 7.275
    item osd.1 weight 7.275
}
host node-2 {
    id -4
    alg straw2
    hash 0
    item osd.2 weight 7.275
    item osd.3 weight 7.275
}
rack rack-a {
    id -10
    alg straw2
    hash 0
    item node-1 weight 14.550
    item node-2 weight 14.550
}
rack rack-b {
    id -11
    alg straw2
    hash 0
    item node-3 weight 14.550
    item node-4 weight 14.550
}
root default {
    id -1
    alg straw2
    hash 0
    item rack-a weight 29.100
    item rack-b weight 29.100
}

// 在不同 Rack 上放置 3 副本
rule replicated_rack {
    id 1
    type replicated
    min_size 1
    max_size 10
    step take default
    step chooseleaf firstn 0 type rack
    step emit
}

// 在不同 Host 上放置 EC 6+2
rule ec_host {
    id 2
    type erasure
    min_size 6
    max_size 8
    step set_chooseleaf_tries 5
    step set_choose_tries 100
    step take default
    step choose indep 0 type host
    step choose indep 8 type osd
    step emit
}

# 导出当前 CRUSH Map
ceph osd getcrushmap -o /tmp/crushmap.bin
crushtool -d /tmp/crushmap.bin -o /tmp/crushmap.txt

# 编辑 /tmp/crushmap.txt 后的注入
crushtool -c /tmp/crushmap.txt -o /tmp/crushmap_new.bin
ceph osd setcrushmap -i /tmp/crushmap_new.bin

# 验证 PG 分布
ceph osd df tree | sort -nk4

[osd]
osd_mclock_profile = high_client_ops       # 生产推荐
# 可选: high_recovery_ops / balanced / high_client_ops
osd_mclock_profile_custom_parameters = "reservation,weight,limit"
osd_mclock_max_capacity_iops_hdd = 315     # HDD 默认 IOPS 容量
osd_mclock_max_capacity_iops_ssd = 21500   # NVMe 默认 IOPS 容量

[osd]
osd_scrub_begin_hour = 0              # Scrub 允许开始时间（0 点）
osd_scrub_end_hour = 6                # Scrub 允许结束时间（凌晨 6 点）
osd_scrub_sleep = 0.1                 # Scrub 间隔休眠（秒），降低对业务影响
osd_deep_scrub_interval = 604800      # Deep-Scrub 间隔（7 天，秒）
osd_scrub_max_interval = 2592000      # 最大 Scrub 间隔（30 天）
osd_deep_scrub_stride = 524288        # Deep-Scrub 一次读取块大小（字节）
osd_max_scrubs = 1                    # 单 OSD 同时进行的 Scrub 数
osd_scrub_auto_repair = false         # 是否自动修复 Scrub 错误（默认仅告警）

# 查看 Scrub 错误
ceph pg dump | grep -E "scrub|inconsistent"

# 查看 PG 的 Deep Scrub 时间戳
ceph pg dump pgs | awk '{print $1, $20, $21}'

# 手动触发 Scrub
ceph pg scrub <pgid>
ceph pg deep-scrub <pgid>

[mon]
mon_osd_down_out_interval = 600        # OSD down 后标记 out 的延迟（秒）
mon_osd_report_timeout = 900           # MON 等待 OSD 上报的超时

[global]
osd_heartbeat_grace = 20               # OSD 心跳宽限时间（秒）
osd_heartbeat_interval = 6             # OSD 心跳间隔（秒）

# 1. 检查 OSD 部署情况
kubectl -n rook-ceph get pods -l app=rook-ceph-osd -o wide

# 2. 如节点彻底故障，清理该节点 OSD 以提高恢复速度
kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- \
  ceph osd crush remove <node-name>

# 3. 如节点预计长时间不可用，提前手动标记 OSD out
ceph osd out osd.<N>    # 手动 out，不等待 mon_osd_down_out_interval

# 4. 节点恢复后重新加入
#    Rook Agent 自动检测设备并重建 OSD

# 1. 确认 OSD 震荡模式
ceph osd tree | grep -E "up|down"
ceph -w | grep -i osd

# 2. 查看 OSD 日志
kubectl -n rook-ceph logs rook-ceph-osd-<N> --tail=200 | grep -iE "heartbeat|timeout"

# 3. 检查节点资源
kubectl top node <node-name>

# 4. 检查网络延迟
ping -c 100 <osd-node-ip> | grep avg

# 1. 查看 stuck PG
ceph pg dump_stuck inactive
ceph pg dump_stuck unclean
ceph pg dump_stuck stale

# 2. 查询特定 PG 的详细信息
ceph pg <pgid> query | jq '.state, .blocked_by, .peering_blocked_by'

# 3. 检查 OSD 是否 up 且 in
ceph osd tree

# 4. 检查 MON 是否形成 quorum
ceph mon stat

# 1. 查看慢请求
ceph daemon osd.<N> ops | jq '.slow_ops'

# 2. 查看 OSD 延迟
ceph osd perf

# 3. 查看实时 I/O
ceph -w | grep -i slow

# 4. 分析具体阻塞原因
ceph daemon osd.<N> dump_historic_ops | jq '.ops[] | select(.description | contains("blocked"))'

apiVersion: ceph.rook.io/v1
kind: CephCluster
spec:
  # 版本锁定
  cephVersion:
    image: quay.io/ceph/ceph:v18.2.1
    allowUnsupported: false

  # 节点亲和性：MON 跨 AZ
  placement:
    mon:
      podAntiAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchLabels:
                app: rook-ceph-mon
            topologyKey: kubernetes.io/hostname
      topologySpreadConstraints:
        - maxSkew: 1
          topologyKey: topology.kubernetes.io/zone
          whenUnsatisfiable: DoNotSchedule

  # PG Autoscaler
  mgr:
    modules:
      - name: pg_autoscaler
        enabled: true

  # 存储：使用 local-storage PV
  storage:
    useAllNodes: false        # 关闭自动发现
    nodes:
      - name: node-1
        devices:
          - name: /dev/nvme0n1
            config:
              osdsPerDevice: "1"
              deviceClass: nvme

  # 健康检查
  healthCheck:
    daemonHealth:
      mon:
        interval: 45s
      osd:
        interval: 60s
    livenessProbe:
      mon:
        periodSeconds: 10
      mgr:
        periodSeconds: 10
      osd:
        periodSeconds: 10

  # 日志保留
  logCollector:
    enabled: true
    periodicity: daily
    maxLogSize: 500M

  # 监控集成
  monitoring:
    enabled: true

# 热数据池：SSD / NVMe，3 副本，高性能
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
  name: fast-pool
spec:
  deviceClass: ssd
  failureDomain: host
  replicated:
    size: 3
    requireSafeReplicaSize: true

# 温数据池：HDD，EC 6+2，大容量低成本
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
  name: archive-pool
spec:
  deviceClass: hdd
  failureDomain: rack
  erasureCoded:
    dataChunks: 6
    codingChunks: 2