Elasticsearch on Kubernetes

# Elasticsearch 索引创建 — 指定分片与副本
PUT /logs-2026.05
{
  "settings": {
    "number_of_shards": 5,
    "number_of_replicas": 2
  },
  "mappings": {
    "properties": {
      "timestamp": { "type": "date" },
      "message":    { "type": "text" },
      "level":      { "type": "keyword" }
    }
  }
}

// 自定义路由 — 写入时指定 routing 参数
POST /orders/_doc?routing=user_42
{
  "order_id": "ORD-2026-0001",
  "user_id": "user_42",
  "amount": 299.99
}

// 查询时需携带相同 routing，否则遍历全部分片
GET /orders/_search?routing=user_42
{
  "query": { "term": { "user_id": "user_42" } }
}

倒排索引构建示例：

文档 1: "Elasticsearch is a search engine"
文档 2: "Apache Lucene powers Elasticsearch"
文档 3: "search engine based on Lucene"

倒排索引：
Term            | (DocID, TF, Positions)
-----------------------------------------
Elasticsearch   | (1,1,[0]), (2,1,[2])
Apache          | (2,1,[0])
Lucene          | (2,1,[1]), (3,1,[3])
search          | (1,1,[3]), (3,1,[0])
engine          | (1,1,[4]), (3,1,[1])

写入流程时序：
Client -> Coordinating Node -> Primary Shard -> [Ingest Pipeline]
                                              -> Memory Buffer + Translog
                                              -> Parallel Replicate to Replicas
                                              -> (Refresh) New Segment (1s)
                                              -> (Flush) fsync + Clear Translog

# 写性能相关索引设置
PUT /my-index/_settings
{
  "index": {
    "refresh_interval": "30s",          # 降低 Refresh 频率提升写入吞吐
    "translog.durability": "async",     # 异步刷盘，提升性能但存在丢失风险
    "translog.sync_interval": "5s",     # 异步模式下 Translog 同步间隔
    "translog.flush_threshold_size": "1gb"
  }
}

// Query 阶段 — 分片级别的查询请求
GET /orders/_search
{
  "query": { "match": { "status": "shipped" } },
  "sort": [{ "order_date": "desc" }, { "_id": "asc" }],
  "size": 100
}

// Search After — 深度分页
GET /orders/_search
{
  "size": 100,
  "query": { "match": { "status": "shipped" } },
  "sort": [
    { "order_date": "desc" },
    { "_id": "asc" }
  ],
  "search_after": ["2026-01-15T10:30:00", "order_12345"]
}

// Scroll — 全量导出
POST /orders/_search?scroll=5m
{
  "size": 1000,
  "query": { "match_all": {} }
}

// Sliced Scroll — 并行导出
GET /orders/_search?scroll=5m
{
  "slice": { "id": 0, "max": 4 },
  "size": 1000,
  "query": { "match_all": {} }
}

# ECK Operator 安装
kubectl create -f https://download.elastic.co/downloads/eck/2.16.1/crds.yaml
kubectl apply -f https://download.elastic.co/downloads/eck/2.16.1/operator.yaml

# ECK CRD 列表
kubectl api-resources --api-group elasticsearch.k8s.elastic.co
# NAME                 SHORTNAMES   APIGROUP                     NAMESPACED   KIND
# agents                            agent.k8s.elastic.co         true         Agent
# apmservers                        apm.k8s.elastic.co           true         ApmServer
# beats                             beat.k8s.elastic.co          true         Beat
# elasticsearches       es          elasticsearch.k8s.elastic.co true         Elasticsearch
# enterprisesearches    ents        enterprisesearch.k8s.elastic.co true      EnterpriseSearch
# kibanas               kb          kibana.k8s.elastic.co        true         Kibana
# logstashes            ls          logstash.k8s.elastic.co      true         Logstash
# elasticmapsservers    ems         maps.k8s.elastic.co          true         ElasticMapsServer

apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
  name: production
spec:
  version: 8.17.0
  nodeSets:
    # Master 专用节点池
    - name: master
      count: 3
      config:
        node.roles: ["master"]
        node.store.allow_mmap: false
      podTemplate:
        spec:
          containers:
            - name: elasticsearch
              resources:
                requests:
                  memory: "2Gi"
                  cpu: "1"
                limits:
                  memory: "2Gi"
                  cpu: "2"
      volumeClaimTemplates:
        - metadata:
            name: elasticsearch-data
          spec:
            accessModes: ["ReadWriteOnce"]
            resources:
              requests:
                storage: 10Gi

    # Data Hot 节点池
    - name: data-hot
      count: 6
      config:
        node.roles: ["data_hot"]
        node.attr.data: "hot"
      podTemplate:
        spec:
          containers:
            - name: elasticsearch
              resources:
                requests:
                  memory: "16Gi"
                  cpu: "4"
                limits:
                  memory: "16Gi"
                  cpu: "8"
      volumeClaimTemplates:
        - metadata:
            name: elasticsearch-data
          spec:
            accessModes: ["ReadWriteOnce"]
            storageClassName: "ssd"
            resources:
              requests:
                storage: 1Ti

    # Ingest 节点池
    - name: ingest
      count: 2
      config:
        node.roles: ["ingest"]
      podTemplate:
        spec:
          containers:
            - name: elasticsearch
              resources:
                requests:
                  memory: "4Gi"
                  cpu: "2"
                limits:
                  memory: "4Gi"
                  cpu: "4"

// ILM Policy — Hot -> Warm -> Cold -> Delete
PUT _ilm/policy/logs-policy
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_primary_shard_size": "50gb",
            "max_age": "1d"
          },
          "set_priority": { "priority": 100 }
        }
      },
      "warm": {
        "min_age": "3d",
        "actions": {
          "allocate": {
            "require": { "data": "warm" }
          },
          "forcemerge": { "max_num_segments": 1 },
          "shrink": { "number_of_shards": 1 },
          "set_priority": { "priority": 50 }
        }
      },
      "cold": {
        "min_age": "30d",
        "actions": {
          "allocate": {
            "require": { "data": "cold" }
          },
          "searchable_snapshot": {
            "snapshot_repository": "s3-backup"
          },
          "set_priority": { "priority": 0 }
        }
      },
      "delete": {
        "min_age": "90d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}

# ECK — 热温冷节点配置
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
spec:
  nodeSets:
    - name: hot
      count: 3
      config:
        node.roles: ["data_hot"]
        node.attr.data: "hot"
      volumeClaimTemplates:
        - spec:
            storageClassName: "ssd"
            resources:
              requests:
                storage: 500Gi
    - name: warm
      count: 3
      config:
        node.roles: ["data_warm"]
        node.attr.data: "warm"
      volumeClaimTemplates:
        - spec:
            storageClassName: "hdd"
            resources:
              requests:
                storage: 2Ti
    - name: cold
      count: 2
      config:
        node.roles: ["data_cold"]
        node.attr.data: "cold"
      volumeClaimTemplates:
        - spec:
            storageClassName: "hdd"
            resources:
              requests:
                storage: 5Ti

apiVersion: kibana.k8s.elastic.co/v1
kind: Kibana
metadata:
  name: production
spec:
  version: 8.17.0
  count: 2
  elasticsearchRef:
    name: production
    namespace: elastic-stack
  config:
    server.publicBaseUrl: https://kibana.example.com
    xpack.fleet.agents.elasticsearch.hosts:
      - https://production-es-http:9200
  podTemplate:
    spec:
      containers:
        - name: kibana
          resources:
            requests:
              memory: 2Gi
              cpu: 1
            limits:
              memory: 4Gi
              cpu: 2
  http:
    tls:
      selfSignedCertificate:
        disabled: true
# Ingress 暴露
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: kibana-ingress
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-prod
spec:
  tls:
    - hosts:
        - kibana.example.com
      secretName: kibana-tls
  rules:
    - host: kibana.example.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: production-kb-http
                port:
                  number: 5601

# Dashboard 导入导出
# 导出
curl -X POST "https://kibana.example.com/api/saved_objects/_export" \
  -H "kbn-xsrf: true" \
  -H "Content-Type: application/json" \
  -d '{ "type": "dashboard", "includeReferencesDeep": true }' \
  -o dashboards.ndjson

# 导入
curl -X POST "https://kibana.example.com/api/saved_objects/_import?overwrite=true" \
  -H "kbn-xsrf: true" \
  --form file=@dashboards.ndjson

# Filebeat Sidecar — 采集应用日志
apiVersion: beat.k8s.elastic.co/v1beta1
kind: Beat
metadata:
  name: app-filebeat
spec:
  type: filebeat
  version: 8.17.0
  elasticsearchRef:
    name: production
  config:
    filebeat.inputs:
      - type: container
        paths:
          - /var/log/containers/*.log
        processors:
          - add_kubernetes_metadata:
              host: ${NODE_NAME}
              matchers:
                - logs_path:
                    logs_path: "/var/log/containers/"
    output.elasticsearch:
      hosts: ["${ECK_ES_HOSTS}"]
      username: "${ECK_ES_USER}"
      password: "${ECK_ES_PASSWORD}"
  daemonSet:
    podTemplate:
      spec:
        serviceAccountName: filebeat
        containers:
          - name: filebeat
            securityContext:
              runAsUser: 0
            volumeMounts:
              - name: varlog
                mountPath: /var/log
            env:
              - name: NODE_NAME
                valueFrom:
                  fieldRef:
                    fieldPath: spec.nodeName
        volumes:
          - name: varlog
            hostPath:
              path: /var/log

# Metricbeat — 采集 ES 集群指标
apiVersion: beat.k8s.elastic.co/v1beta1
kind: Beat
metadata:
  name: metricbeat
spec:
  type: metricbeat
  version: 8.17.0
  elasticsearchRef:
    name: production
  kibanaRef:
    name: production
  config:
    metricbeat.modules:
      - module: elasticsearch
        metricsets: ["node", "node_stats", "index", "cluster_stats"]
        period: 10s
        hosts: ["https://${ECK_ES_HOSTS}:9200"]
        username: "${ECK_ES_USER}"
        password: "${ECK_ES_PASSWORD}"
        ssl.verification_mode: "certificate"
      - module: kubernetes
        metricsets: ["node", "system", "pod", "container", "volume"]
        period: 10s
        host: ${NODE_NAME}
    output.elasticsearch:
      hosts: ["https://${ECK_ES_HOSTS}:9200"]
      username: "${ECK_ES_USER}"
      password: "${ECK_ES_PASSWORD}"
  deployment:
    podTemplate:
      spec:
        serviceAccountName: metricbeat

apiVersion: apm.k8s.elastic.co/v1
kind: ApmServer
metadata:
  name: production
spec:
  version: 8.17.0
  count: 3
  elasticsearchRef:
    name: production
  kibanaRef:
    name: production
  config:
    apm-server:
      host: "0.0.0.0:8200"
      secret_token: "${APM_SECRET_TOKEN}"
      rum:
        enabled: true
        allow_origins: ["https://app.example.com"]
        source_mapping:
          cache.expiration: 5m
          index_pattern: "apm-*-sourcemap*"
      sampling:
        tail:
          enabled: true
          interval: 1m
          policies:
            - sample_rate: 0.1
            - sample_rate: 1.0
              trace.name: "checkout"
  http:
    tls:
      selfSignedCertificate:
        disabled: true
  podTemplate:
    spec:
      containers:
        - name: apm-server
          resources:
            requests:
              memory: 512Mi
              cpu: 500m
            limits:
              memory: 1Gi
              cpu: 1

// Java APM Agent 配置示例
// -javaagent:elastic-apm-agent.jar \
//   -Delastic.apm.server_url=https://apm.example.com \
//   -Delastic.apm.secret_token=${APM_SECRET_TOKEN} \
//   -Delastic.apm.service_name=order-service \
//   -Delastic.apm.environment=production \
//   -Delastic.apm.application_packages=com.example.order

graph TD
    ES[Elasticsearch CR<br/>production] -->|elasticsearchRef| KB[Kibana CR<br/>production]
    ES -->|elasticsearchRef| APM[ApmServer CR<br/>production]
    ES -->|elasticsearchRef| BEAT[Beat CR<br/>filebeat / metricbeat]
    APM -->|kibanaRef| KB
    BEAT -->|kibanaRef| KB

# 完整 Elastic Stack 部署示例
# Elasticsearch
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
  name: production
  namespace: elastic-stack
spec:
  version: 8.17.0
  nodeSets:
    - name: default
      count: 3
      config:
        node.store.allow_mmap: false
      volumeClaimTemplates:
        - metadata:
            name: elasticsearch-data
          spec:
            resources:
              requests:
                storage: 200Gi

# Kibana
apiVersion: kibana.k8s.elastic.co/v1
kind: Kibana
metadata:
  name: production
  namespace: elastic-stack
spec:
  version: 8.17.0
  count: 1
  elasticsearchRef:
    name: production

# APM Server
apiVersion: apm.k8s.elastic.co/v1
kind: ApmServer
metadata:
  name: production
  namespace: elastic-stack
spec:
  version: 8.17.0
  count: 2
  elasticsearchRef:
    name: production

# Filebeat DaemonSet
apiVersion: beat.k8s.elastic.co/v1beta1
kind: Beat
metadata:
  name: filebeat
  namespace: elastic-stack
spec:
  type: filebeat
  version: 8.17.0
  elasticsearchRef:
    name: production
  kibanaRef:
    name: production
  config:
    filebeat.inputs:
      - type: container
        paths: ["/var/log/containers/*.log"]
    output.elasticsearch:
      hosts: ["https://${ECK_ES_HOSTS}:9200"]
  daemonSet: {}

# ES 7.x — Zen Discovery 配置
discovery.zen.minimum_master_nodes: 2    # 3 个 Master 节点的 Quorum
discovery.zen.ping.unicast.hosts:
  - es-master-0.es-master-headless
  - es-master-1.es-master-headless
  - es-master-2.es-master-headless

# ES 8.x — 无需 minimum_master_nodes
# ECK 自动配置 voting 节点
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
spec:
  version: 8.17.0
  nodeSets:
    - name: master
      count: 3
      config:
        node.roles: ["master"]

# 查看集群健康状态
GET /_cluster/health?level=shards
{
  "cluster_name": "production",
  "status": "yellow",
  "timed_out": false,
  "number_of_nodes": 6,
  "active_primary_shards": 128,
  "active_shards": 250,
  "unassigned_shards": 6,
  "relocating_shards": 0,
  "initializing_shards": 0,
  "number_of_pending_tasks": 0,
  "number_of_in_flight_fetch": 0
}

# 查看 Red 分片原因
GET /_cluster/allocation/explain
{
  "index": "logs-2026.05.15",
  "shard": 2,
  "primary": true,
  "current_state": "unassigned",
  "unassigned_info": {
    "reason": "NODE_LEFT",
    "details": "node [node-3] left the cluster"
  },
  "can_allocate": "no",
  "allocate_explanation": "no valid shard copies exist"
}

# ECK — 分片分配感知配置
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
spec:
  version: 8.17.0
  nodeSets:
    - name: data
      count: 6
      config:
        cluster.routing.allocation.awareness.attributes: zone
        cluster.routing.allocation.awareness.force.zone.values: ap-southeast-1a,ap-southeast-1b,ap-southeast-1c
      podTemplate:
        spec:
          topologySpreadConstraints:
            - maxSkew: 1
              topologyKey: topology.kubernetes.io/zone
              whenUnsatisfiable: DoNotSchedule
              labelSelector:
                matchLabels:
                  elasticsearch.k8s.elastic.co/cluster-name: production

# 验证分片分布
GET /_cat/shards?v&h=index,shard,prirep,state,node,unassigned.reason

// 注册 S3 Repository
PUT _snapshot/s3-backup
{
  "type": "s3",
  "settings": {
    "bucket": "es-snapshots-production",
    "region": "ap-southeast-1",
    "base_path": "production-cluster",
    "max_snapshot_bytes_per_sec": "100mb",
    "max_restore_bytes_per_sec": "100mb"
  }
}

// 创建快照
PUT _snapshot/s3-backup/snapshot-2026-05-26
{
  "indices": "logs-*,metrics-*",
  "ignore_unavailable": true,
  "include_global_state": false,
  "partial": false
}

// 恢复快照
POST _snapshot/s3-backup/snapshot-2026-05-26/_restore
{
  "indices": "logs-2026.05.*",
  "ignore_unavailable": true,
  "rename_pattern": "logs-(.+)",
  "rename_replacement": "restored-logs-$1"
}

# ECK — 注册 S3 Snapshot Repository
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
  name: production
spec:
  version: 8.17.0
  secureSettings:
    - secretName: s3-credentials
  podTemplate:
    spec:
      containers:
        - name: elasticsearch
          env:
            - name: S3_ACCESS_KEY
              valueFrom:
                secretKeyRef:
                  name: s3-credentials
                  key: access_key
            - name: S3_SECRET_KEY
              valueFrom:
                secretKeyRef:
                  name: s3-credentials
                  key: secret_key
# 注册 Repository（通过 Kibana DevTools 或 Curator 调用）

# ECK — RBAC Role 定义
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
  name: production
spec:
  version: 8.17.0
  auth:
    roles:
      - secretName: es-roles
# Secret 内容 — 自定义 Role
apiVersion: v1
kind: Secret
metadata:
  name: es-roles
stringData:
  log-reader-role.yml: |
    log_reader:
      cluster: ["monitor"]
      indices:
        - names: ["logs-*"]
          privileges: ["read", "view_index_metadata"]
        - names: ["metrics-*"]
          privileges: ["read"]
      applications:
        - application: "kibana-.kibana"
          privileges: ["feature_discover.read"]
          resources: ["space:default"]    
  admin-role.yml: |
    admin:
      cluster: ["all"]
      indices:
        - names: ["*"]
          privileges: ["all"]
      applications:
        - application: "kibana-*"
          privileges: ["*"]
          resources: ["*"]    

// 通过 API 创建用户并绑定角色
POST /_security/user/logstash_writer
{
  "password": "secure_password",
  "roles": ["log_writer"],
  "full_name": "Logstash Writer"
}

// Document-Level Security — 按角色过滤文档
POST /_security/role/user_read_own
{
  "indices": [
    {
      "names": ["orders-*"],
      "privileges": ["read"],
      "query": {
        "template": {
          "source": "{\"term\": {\"user_id\": \"XQOPEN_user.usernameXQCLOSE\"}}"
        }
      }
    }
  ]
}

// Field-Level Security — 按角色过滤字段
POST /_security/role/log_reader_no_pii
{
  "indices": [
    {
      "names": ["orders-*"],
      "privileges": ["read"],
      "field_security": {
        "grant": ["order_id", "status", "amount", "created_at"],
        "except": ["credit_card", "ssn"]
      }
    }
  ]
}

// Mapping 优化示例
PUT /optimized_orders
{
  "settings": {
    "number_of_shards": 5,
    "number_of_replicas": 1,
    "refresh_interval": "30s"
  },
  "mappings": {
    "_source": {
      "excludes": ["large_raw_payload"]
    },
    "properties": {
      "order_id": {
        "type": "keyword",
        "doc_values": true
      },
      "status": {
        "type": "keyword"
      },
      "description": {
        "type": "text",
        "index_options": "freqs",
        "norms": false
      },
      "amount": {
        "type": "scaled_float",
        "scaling_factor": 100
      },
      "tags": {
        "type": "keyword",
        "index": false
      }
    }
  }
}

// 查询优化 — filter 上下文优先
GET /orders/_search
{
  "query": {
    "bool": {
      "filter": [
        { "term": { "status": "shipped" } },
        { "range": { "amount": { "gte": 100, "lte": 500 } } }
      ],
      "must": [
        { "match": { "description": "express delivery" } }
      ]
    }
  }
}

// Bulk API 写入
POST _bulk
{ "index": { "_index": "orders", "_id": "1" } }
{ "user_id": "42", "amount": 99.99, "status": "pending" }
{ "index": { "_index": "orders", "_id": "2" } }
{ "user_id": "43", "amount": 199.99, "status": "shipped" }
{ "update": { "_index": "orders", "_id": "1" } }
{ "doc": { "status": "cancelled" } }
{ "delete": { "_index": "orders", "_id": "2" } }

// 写入优化索引设置
PUT /high-throughput-logs/_settings
{
  "index": {
    "refresh_interval": "30s",
    "number_of_replicas": 0,               # 写入时 0 副本
    "translog.durability": "async",
    "translog.sync_interval": "10s",
    "translog.flush_threshold_size": "2gb"
  }
}

// 写入完成后增加 Replica
PUT /high-throughput-logs/_settings
{
  "index": {
    "number_of_replicas": 1
  }
}

# 客户端 Bulk 写入最佳参数
# Node.js Elasticsearch Client 示例
const result = await client.helpers.bulk({
  datasource: docs,          // 文档生成器
  onDocument(doc) {
    return { index: { _index: 'orders' } };
  },
  refreshOnCompletion: false, // 客户端不等待 Refresh
  concurrency: 4,             // 并发批次
  flushBytes: 5 * 1024 * 1024 // 每达到 5MB 刷新一次
});

# JVM 堆内存配置 (jvm.options)
-Xms8g
-Xmx8g

# 启用 G1GC
-XX:+UseG1GC
-XX:MaxGCPauseMillis=200
-XX:G1HeapRegionSize=4m
-XX:InitiatingHeapOccupancyPercent=30

# GC 日志
-Xlog:gc*,gc+age=trace:file=/var/log/elasticsearch/gc.log:time,uptime,level,tags:filecount=10,filesize=50m

# ECK — JVM Heap 配置
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
spec:
  version: 8.17.0
  nodeSets:
    - name: data-hot
      count: 6
      config:
        # 通过环境变量设置 Heap 大小
      podTemplate:
        spec:
          containers:
            - name: elasticsearch
              env:
                - name: ES_JAVA_OPTS
                  value: "-Xms16g -Xmx16g"
              resources:
                requests:
                  memory: "32Gi"
                  cpu: "8"
                limits:
                  memory: "32Gi"
                  cpu: "8"

// 监控 Circuit Breaker 状态
GET /_nodes/stats/breaker
{
  "nodes": {
    "node-1": {
      "breakers": {
        "parent": {
          "limit_size_in_bytes": 17179869184,    // 16GB
          "limit_size": "16gb",
          "estimated_size_in_bytes": 2147483648,  // 2GB
          "estimated_size": "2gb",
          "tripped": 0
        },
        "fielddata": {
          "limit_size_in_bytes": 6871947673,     // 40% of parent
          "tripped": 0
        },
        "request": {
          "limit_size_in_bytes": 10307921510,    // 60% of parent
          "tripped": 0
        }
      }
    }
  }
}

// 完整 ILM Policy
PUT _ilm/policy/hot-warm-cold-delete
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_primary_shard_size": "50gb",
            "max_age": "1d",
            "max_docs": 100000000
          },
          "set_priority": { "priority": 100 }
        }
      },
      "warm": {
        "min_age": "3d",
        "actions": {
          "allocate": {
            "number_of_replicas": 1,
            "require": { "data": "warm" }
          },
          "forcemerge": {
            "max_num_segments": 1
          },
          "shrink": {
            "number_of_shards": 1
          },
          "set_priority": { "priority": 50 }
        }
      },
      "cold": {
        "min_age": "30d",
        "actions": {
          "allocate": {
            "number_of_replicas": 0,
            "require": { "data": "cold" }
          },
          "searchable_snapshot": {
            "snapshot_repository": "s3-snapshots"
          },
          "set_priority": { "priority": 0 }
        }
      },
      "delete": {
        "min_age": "90d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}

// 绑定到 Index Template
PUT _index_template/logs-template
{
  "index_patterns": ["logs-*"],
  "template": {
    "settings": {
      "index.lifecycle.name": "hot-warm-cold-delete",
      "index.lifecycle.rollover_alias": "logs"
    }
  }
}

// 创建初始索引并绑定 Rollover Alias
PUT logs-000001
{
  "aliases": {
    "logs": {
      "is_write_index": true
    }
  }
}

// Follower 集群 — 创建 Remote Cluster 连接
PUT _cluster/settings
{
  "persistent": {
    "cluster": {
      "remote": {
        "leader-cluster": {
          "seeds": ["10.0.1.10:9300", "10.0.1.11:9300"]
        }
      }
    }
  }
}

// 创建 Follower Index
PUT /orders/_ccr/follow
{
  "remote_cluster": "leader-cluster",
  "leader_index": "orders",
  "max_read_request_operation_count": 5000,
  "max_outstanding_read_requests": 12,
  "max_read_request_size": "32mb",
  "max_write_request_operation_count": 5000,
  "max_write_request_size": "9223372036854775807b",
  "max_outstanding_write_requests": 9,
  "read_poll_timeout": "1m"
}

// 自动跟随配置
PUT /_ccr/auto_follow/logs-auto-follow
{
  "remote_cluster": "leader-cluster",
  "leader_index_patterns": ["logs-*", "metrics-*"],
  "follow_index_pattern": "XQOPENleader_indexXQCLOSE-follower",
  "max_outstanding_read_requests": 12,
  "max_outstanding_write_requests": 9,
  "max_write_request_operation_count": 5000
}

# ECK — Multi-Cluster CCR 部署架构
# Cluster A (Asia): Leader — 主写入
# Cluster B (Europe): Follower — 只读灾备

# Cluster B 配置 Remote Cluster 连接
# 使用 ECK Trust Relationship 建立集群间信任
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
  name: cluster-b
spec:
  version: 8.17.0
  remoteClusters:
    - name: cluster-a
      elasticsearchRef:
        name: cluster-a
        namespace: elastic-stack-asia

// 配置 Remote Cluster 连接
PUT _cluster/settings
{
  "persistent": {
    "cluster": {
      "remote": {
        "asia-cluster": {
          "mode": "sniff",
          "seeds": ["10.0.1.10:9300", "10.0.1.11:9300"],
          "transport.compress": true,
          "skip_unavailable": true
        },
        "europe-cluster": {
          "mode": "proxy",
          "proxy_address": "europe-gateway:9300",
          "skip_unavailable": false
        }
      }
    }
  }
}

// 跨集群搜索
GET /asia-cluster:logs-*,europe-cluster:logs-*,local-logs/_search
{
  "query": {
    "bool": {
      "must": [
        { "range": { "@timestamp": { "gte": "now-1h" } } },
        { "match": { "level": "ERROR" } }
      ]
    }
  },
  "size": 100,
  "ccs_minimize_roundtrips": true
}

// Kibana 中创建 Cross-Cluster Index Pattern
// Management -> Stack Management -> Kibana -> Index Patterns
// Index Pattern: asia-cluster:logs-*,europe-cluster:logs-*,local-logs

# ECK — 滚动升级配置
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
  name: production
spec:
  version: 8.17.0              # 修改此字段触发滚动升级
  updateStrategy:
    changeBudget:
      maxUnavailable: 1        # 同时最多不可用节点数
      maxSurge: 0              # 滚动升级不允许临时超出
  nodeSets:
    - name: data
      count: 6
      config:
        # 动态配置变更（不需重启）
        cluster.routing.allocation.disk.watermark.low: 85%
        cluster.routing.allocation.disk.watermark.high: 90%

# 手动升级流程（无 ECK 场景）
# 1. 暂停分片分配
PUT _cluster/settings
{
  "persistent": {
    "cluster.routing.allocation.enable": "primaries"
  }
}

# 2. 停止 Node-1，升级，启动

# 3. 恢复分片分配
PUT _cluster/settings
{
  "persistent": {
    "cluster.routing.allocation.enable": "all"
  }
}

# 4. 等待 Green
GET _cluster/health?wait_for_status=green&timeout=30m

# 5. 循环节点 2-N

# elasticsearch_exporter Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: es-exporter
spec:
  replicas: 1
  selector:
    matchLabels:
      app: es-exporter
  template:
    metadata:
      labels:
        app: es-exporter
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "9114"
    spec:
      containers:
        - name: exporter
          image: quay.io/prometheuscommunity/elasticsearch-exporter:v1.8.0
          args:
            - "--es.uri=https://production-es-http:9200"
            - "--es.ssl-skip-verify"
            - "--es.all"
            - "--es.indices"
            - "--es.indices_settings"
            - "--es.shards"
            - "--es.slm"
            - "--es.data_stream"
          env:
            - name: ES_USERNAME
              valueFrom:
                secretKeyRef:
                  name: es-monitoring-user
                  key: username
            - name: ES_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: es-monitoring-user
                  key: password
          ports:
            - containerPort: 9114
              name: metrics

# Prometheus 告警规则示例
groups:
  - name: elasticsearch
    rules:
      - alert: ESClusterRed
        expr: elasticsearch_cluster_health_status{color="red"} == 1
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "ES cluster XQOPEN $labels.cluster XQCLOSE is RED"

      - alert: ESHeapUsageHigh
        expr: elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"} > 0.85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "ES node XQOPEN $labels.node XQCLOSE heap usage > 85%"

      - alert: ESUnassignedShards
        expr: elasticsearch_cluster_health_unassigned_shards > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "ES cluster has XQOPEN $value XQCLOSE unassigned shards"

// 创建自定义 Role — 日志读取角色
POST /_security/role/log_reader
{
  "cluster": ["monitor", "read_ilm"],
  "indices": [
    {
      "names": ["logs-*"],
      "privileges": ["read", "view_index_metadata"],
      "allow_restricted_indices": false
    }
  ],
  "applications": [
    {
      "application": "kibana-.kibana",
      "privileges": ["feature_discover.read"],
      "resources": ["space:default"]
    }
  ]
}

// 创建自定义 Role — 含 DLS + FLS
POST /_security/role/order_viewer_dept
{
  "cluster": [],
  "indices": [
    {
      "names": ["orders-*"],
      "privileges": ["read"],
      "query": {
        "template": {
          "source": "{\"term\": {\"department\": \"XQOPEN_user.metadata.departmentXQCLOSE\"}}"
        }
      },
      "field_security": {
        "grant": ["order_id", "amount", "status", "created_at", "department"]
      }
    }
  ]
}

// 创建用户并绑定角色
POST /_security/user/alice
{
  "password": "secure_password",
  "roles": ["log_reader", "order_viewer_dept"],
  "full_name": "Alice Smith",
  "email": "alice@example.com",
  "metadata": {
    "department": "engineering"
  }
}

// 创建 API Key（应用访问）
POST /_security/api_key
{
  "name": "order-service-key",
  "role_descriptors": {
    "order_writer": {
      "cluster": ["monitor"],
      "indices": [
        {
          "names": ["orders-*"],
          "privileges": ["create_index", "index", "read"]
        }
      ]
    }
  },
  "expiration": "90d"
}

// SLM Policy — 每日快照保留 30 天
PUT _slm/policy/daily-snapshots
{
  "schedule": "0 1 * * *",
  "name": "<daily-snap-{now/d}>",
  "repository": "s3-backup",
  "config": {
    "indices": ["logs-*", "metrics-*", ".kibana*"],
    "ignore_unavailable": true,
    "include_global_state": false,
    "partial": false
  },
  "retention": {
    "expire_after": "30d",
    "min_count": 5,
    "max_count": 60
  }
}

// SLM Policy — 分层保留（GFS 策略）
PUT _slm/policy/layered-retention
{
  "schedule": "0 2 * * *",
  "name": "<layered-{now/d}>",
  "repository": "s3-backup",
  "config": {
    "indices": ["*,-.monitoring-*"],
    "include_global_state": true
  },
  "retention": {
    "expire_after": "90d",
    "min_count": 7,
    "max_count": 100
  }
}

// 查看 SLM 状态
GET _slm/stats
{
  "retention_runs": 120,
  "retention_failed": 0,
  "retention_timed_out": 0,
  "retention_deletion_time_millis": 45000,
  "policy_stats": [{
    "policy": "daily-snapshots",
    "snapshots_taken": 30,
    "snapshots_failed": 0,
    "snapshots_deleted": 25,
    "retention_deletions": 25
  }]
}

// Component Template — 可复用组件
PUT _component_template/logs-settings
{
  "template": {
    "settings": {
      "number_of_shards": 3,
      "number_of_replicas": 1,
      "refresh_interval": "30s",
      "index.lifecycle.name": "logs-ilm-policy"
    }
  }
}

PUT _component_template/logs-mappings
{
  "template": {
    "mappings": {
      "properties": {
        "@timestamp": { "type": "date" },
        "message": { "type": "text" },
        "level": { "type": "keyword" },
        "host.name": { "type": "keyword" },
        "service.name": { "type": "keyword" },
        "trace.id": { "type": "keyword" }
      }
    }
  }
}

// Composable Index Template — 组合引用 Component
PUT _index_template/logs-template
{
  "priority": 100,
  "index_patterns": ["logs-*"],
  "composed_of": ["logs-settings", "logs-mappings"],
  "data_stream": {},
  "_meta": {
    "description": "Template for application logs data stream"
  }
}

// 创建 Data Stream
PUT _data_stream/logs-app
// 写入 Data Stream
POST /logs-app/_doc
{
  "@timestamp": "2026-05-26T10:00:00Z",
  "message": "User login successful",
  "level": "INFO",
  "host.name": "web-01",
  "service.name": "auth-service"
}

// 设置 Data Stream 为 Time Series 模式（ES 8.x tsdb）
PUT _index_template/metrics-template
{
  "priority": 200,
  "index_patterns": ["metrics-*"],
  "data_stream": {},
  "template": {
    "settings": {
      "index.mode": "time_series",
      "index.routing_path": ["host.name"]
    },
    "mappings": {
      "properties": {
        "@timestamp": { "type": "date" },
        "host.name": {
          "type": "keyword",
          "time_series_dimension": true
        },
        "cpu.usage": {
          "type": "double",
          "time_series_metric": "gauge"
        }
      }
    }
  }
}

# 排查 Unassigned Shards
GET _cluster/allocation/explain
{
  "index": "logs-2026.05.15",
  "shard": 2,
  "primary": true
}

# 手动重分配 (allocate_stale_primary — 数据可能丢失)
POST _cluster/reroute
{
  "commands": [
    {
      "allocate_stale_primary": {
        "index": "logs-2026.05.15",
        "shard": 2,
        "node": "data-node-3",
        "accept_data_loss": true
      }
    }
  ]
}

# 检查 Disk Watermark
GET _cat/allocation?v&h=node,disk.percent,disk.total,disk.used,disk.avail

# 调整 Disk Watermark（紧急）
PUT _cluster/settings
{
  "transient": {
    "cluster.routing.allocation.disk.watermark.high": "95%",
    "cluster.routing.allocation.disk.watermark.flood_stage": "97%"
  }
}

// 查看已触发的断路器
GET _nodes/stats/breaker
// 返回示例 — 关注 tripped > 0 的断路器

// 调整 Field Data 断路器限制
PUT _cluster/settings
{
  "transient": {
    "indices.breaker.fielddata.limit": "50%",
    "indices.breaker.request.limit": "70%",
    "indices.breaker.total.limit": "95%"
  }
}

// 在 Mapping 中禁用 Text 字段的 fielddata
PUT /my-index/_mapping
{
  "properties": {
    "description": {
      "type": "text",
      "fielddata": false
    },
    "description_keyword": {
      "type": "keyword"
    }
  }
}

# 完整生产环境配置示例
apiVersion: elasticsearch.k8s.elastic.co/v1
kind: Elasticsearch
metadata:
  name: production
  namespace: elastic-stack
spec:
  version: 8.17.0
  updateStrategy:
    changeBudget:
      maxUnavailable: 1

  # Master 节点池
  nodeSets:
    - name: master
      count: 3
      config:
        node.roles: ["master"]
        node.store.allow_mmap: false
        xpack.security.authc.accept_default_password: false
      podTemplate:
        spec:
          initContainers:
            - name: sysctl
              securityContext:
                privileged: true
              command:
                - sysctl
                - -w
                - vm.max_map_count=262144
          containers:
            - name: elasticsearch
              env:
                - name: ES_JAVA_OPTS
                  value: "-Xms2g -Xmx2g"
              resources:
                requests:
                  memory: 4Gi
                  cpu: 1
                limits:
                  memory: 4Gi
                  cpu: 2
      volumeClaimTemplates:
        - metadata:
            name: elasticsearch-data
          spec:
            accessModes: ["ReadWriteOnce"]
            resources:
              requests:
                storage: 20Gi

    # Data Hot 节点池
    - name: data-hot
      count: 6
      config:
        node.roles: ["data_hot", "data_content"]
        node.attr.data: "hot"
        node.store.allow_mmap: false
        # 慢查询日志
        index.search.slowlog.threshold.query.warn: 5s
        index.search.slowlog.threshold.query.info: 2s
        index.indexing.slowlog.threshold.index.warn: 5s
      podTemplate:
        spec:
          initContainers:
            - name: sysctl
              securityContext:
                privileged: true
              command: ["sysctl", "-w", "vm.max_map_count=262144"]
          containers:
            - name: elasticsearch
              env:
                - name: ES_JAVA_OPTS
                  value: "-Xms16g -Xmx16g"
              resources:
                requests:
                  memory: 32Gi
                  cpu: 8
                limits:
                  memory: 32Gi
                  cpu: 8
              readinessProbe:
                exec:
                  command:
                    - bash
                    - -c
                    - |
                      curl -s -k -u "elastic:${ELASTIC_PASSWORD}" https://localhost:9200/_cluster/health?local=true | grep -q '"status":"green"'                      
                initialDelaySeconds: 120
                periodSeconds: 30
      volumeClaimTemplates:
        - metadata:
            name: elasticsearch-data
          spec:
            accessModes: ["ReadWriteOnce"]
            storageClassName: "ssd"
            resources:
              requests:
                storage: 1Ti

    # Ingest 节点池
    - name: ingest
      count: 2
      config:
        node.roles: ["ingest"]
      podTemplate:
        spec:
          containers:
            - name: elasticsearch
              env:
                - name: ES_JAVA_OPTS
                  value: "-Xms4g -Xmx4g"
              resources:
                requests:
                  memory: 8Gi
                  cpu: 2
                limits:
                  memory: 8Gi
                  cpu: 4

  # TLS 安全配置
  http:
    tls:
      selfSignedCertificate:
        disabled: true
      certificate:
        secretName: es-http-tls
  transport:
    tls:
      certificate:
        secretName: es-transport-tls

  # 监控集成
  monitoring:
    metrics:
      elasticsearchRefs:
        - name: production
    logs:
      elasticsearchRefs:
        - name: production

# Network Policy — 限制 ES Pod 通信范围
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: es-network-policy
  namespace: elastic-stack
spec:
  podSelector:
    matchLabels:
      elasticsearch.k8s.elastic.co/cluster-name: production
  policyTypes:
    - Ingress
    - Egress
  ingress:
    - from:
        - podSelector: {}
        - namespaceSelector:
            matchLabels:
              name: monitoring
      ports:
        - protocol: TCP
          port: 9200
        - protocol: TCP
          port: 9300
  egress:
    - to:
        - podSelector:
            matchLabels:
              elasticsearch.k8s.elastic.co/cluster-name: production
      ports:
        - protocol: TCP
          port: 9300