Copy# k8s/etcd-statefulset.yaml
apiVersion: v1
kind: Service
metadata:
  name: etcd-headless
  namespace: demo-gateway
  labels:
    app: etcd
spec:
  clusterIP: None
  ports:
    - name: client
      port: 2379
      targetPort: 2379
    - name: peer
      port: 2380
      targetPort: 2380
  selector:
    app: etcd
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: etcd
  namespace: demo-gateway
spec:
  serviceName: etcd-headless
  replicas: 3
  selector:
    matchLabels:
      app: etcd
  podManagementPolicy: Parallel
  template:
    metadata:
      labels:
        app: etcd
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: app
                    operator: In
                    values:
                      - etcd
              topologyKey: kubernetes.io/hostname
      containers:
        - name: etcd
          image: bitnami/etcd:3.5
          env:
            - name: ALLOW_NONE_AUTHENTICATION
              value: "yes"
            - name: ETCD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: ETCD_ADVERTISE_CLIENT_URLS
              value: "http://$(ETCD_NAME).etcd-headless.demo-gateway.svc.cluster.local:2379"
            - name: ETCD_LISTEN_CLIENT_URLS
              value: "http://0.0.0.0:2379"
            - name: ETCD_INITIAL_ADVERTISE_PEER_URLS
              value: "http://$(ETCD_NAME).etcd-headless.demo-gateway.svc.cluster.local:2380"
            - name: ETCD_LISTEN_PEER_URLS
              value: "http://0.0.0.0:2380"
            - name: ETCD_INITIAL_CLUSTER_TOKEN
              value: "demo-etcd-cluster"
            - name: ETCD_INITIAL_CLUSTER
              value: "etcd-0=http://etcd-0.etcd-headless.demo-gateway.svc.cluster.local:2380,etcd-1=http://etcd-1.etcd-headless.demo-gateway.svc.cluster.local:2380,etcd-2=http://etcd-2.etcd-headless.demo-gateway.svc.cluster.local:2380"
            - name: ETCD_INITIAL_CLUSTER_STATE
              value: "new"
          ports:
            - containerPort: 2379
              name: client
            - containerPort: 2380
              name: peer
          livenessProbe:
            exec:
              command:
                - /bin/sh
                - -c
                - etcdctl endpoint health
            initialDelaySeconds: 30
            periodSeconds: 10
          readinessProbe:
            exec:
              command:
                - /bin/sh
                - -c
                - etcdctl endpoint health
            initialDelaySeconds: 10
            periodSeconds: 5
          volumeMounts:
            - name: data
              mountPath: /bitnami/etcd
          resources:
            requests:
              cpu: 200m
              memory: 256Mi
            limits:
              cpu: 1000m
              memory: 1Gi
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        accessModes: ["ReadWriteOnce"]
        resources:
          requests:
            storage: 10Gi

Copy# 检查集群健康
kubectl exec -n demo-gateway etcd-0 -- etcdctl endpoint health --cluster
# 预期: 三个节点都是 healthy
 
# 查看集群成员
kubectl exec -n demo-gateway etcd-0 -- etcdctl member list

Copy# k8s/gateway-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: api-gateway
  namespace: demo-gateway
  labels:
    app: api-gateway
spec:
  replicas: 3
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1          # 最多多创建 1 个 Pod
      maxUnavailable: 0    # 保证至少 3 个 Pod 始终可用
  selector:
    matchLabels:
      app: api-gateway
  template:
    metadata:
      labels:
        app: api-gateway
    spec:
      # ⚠️ 关键1：给足优雅终止时间
      terminationGracePeriodSeconds: 60
 
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              podAffinityTerm:
                labelSelector:
                  matchExpressions:
                    - key: app
                      operator: In
                      values:
                        - api-gateway
                topologyKey: kubernetes.io/hostname
 
      containers:
        - name: gateway
          image: apache/apisix:3.10.0-centos
          ports:
            - containerPort: 9080
              name: proxy
              protocol: TCP
            - containerPort: 9180
              name: admin
              protocol: TCP
            - containerPort: 9443
              name: tls
              protocol: TCP
 
          env:
            - name: APISIX_STAND_ALONE
              value: "false"
            - name: APISIX_DEPLOYMENT_ETCD_HOST
              value: "http://etcd-headless.demo-gateway.svc.cluster.local:2379"
 
          # ⚠️ 关键2：生命周期钩子
          lifecycle:
            preStop:
              exec:
                command:
                  - /bin/sh
                  - -c
                  - |
                    echo "=== Preparing for graceful shutdown ==="
                    # 1. 将自己从负载均衡摘除（等 5 秒让 Service 端点更新）
                    sleep 5
                    # 2. 停止接收新请求（Nginx graceful shutdown）
                    apisix quit || true
                    # 3. 等待存量请求处理完毕
                    sleep 10
                    echo "=== Graceful shutdown completed ==="
 
          # ⚠️ 关键3：就绪探针——旧 Pod 启动后再杀
          readinessProbe:
            httpGet:
              path: /apisix/status
              port: 9080
            initialDelaySeconds: 15
            periodSeconds: 5
            failureThreshold: 3
 
          livenessProbe:
            httpGet:
              path: /apisix/status
              port: 9080
            initialDelaySeconds: 30
            periodSeconds: 15
            failureThreshold: 5
 
          resources:
            requests:
              cpu: 500m
              memory: 512Mi
            limits:
              cpu: 2000m
              memory: 2Gi
 
          volumeMounts:
            - name: gateway-config
              mountPath: /usr/local/apisix/conf/config.yaml
              subPath: config.yaml
 
      volumes:
        - name: gateway-config
          configMap:
            name: gateway-config

Copy# 开一个终端持续压测
while true; do
  curl -s -o /dev/null -w "%{http_code} %{time_total}s\n" \
    http://api-gateway-svc.demo-gateway.svc.cluster.local:9080/apisix/status
  sleep 0.1
done
 
# 另一个终端触发更新
kubectl rollout restart deployment/api-gateway -n demo-gateway
 
# 观察终端 A：全程不应出现 5xx

Copy# 安装 cert-manager
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.14.0/cert-manager.yaml

Copy# k8s/cert-issuer.yaml
# 使用云厂商 DNS 验证的 ClusterIssuer
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: letsencrypt-prod
spec:
  acme:
    server: https://acme-v02.api.letsencrypt.org/directory
    email: demo-admin@example.com
    privateKeySecretRef:
      name: letsencrypt-prod-key
    solvers:
      - http01:
          ingress:
            class: nginx
---
# 为网关域名申请证书
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: gateway-tls-cert
  namespace: demo-gateway
spec:
  secretName: gateway-tls-secret
  duration: 2160h  # 90 天
  renewBefore: 360h # 提前 15 天续期
  dnsNames:
    - api.example.com
    - admin-api.example.com
  issuerRef:
    name: letsencrypt-prod
    kind: ClusterIssuer
---
# Ingress 配置 TLS
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: gateway-ingress
  namespace: demo-gateway
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-prod
    nginx.ingress.kubernetes.io/proxy-body-size: "10m"
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
spec:
  ingressClassName: nginx
  tls:
    - hosts:
        - api.example.com
      secretName: gateway-tls-secret
  rules:
    - host: api.example.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: api-gateway-svc
                port:
                  number: 9080

Copy# k8s/gateway-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: gateway-config
  namespace: demo-gateway
data:
  config.yaml: |
    apisix:
      node_listen: 9080
      enable_admin: true
      enable_admin_cors: true
 
      # etcd 集群地址
      deployment:
        role: traditional
        role_traditional:
          config_provider: etcd
        etcd:
          host:
            - "http://etcd-0.etcd-headless.demo-gateway.svc.cluster.local:2379"
            - "http://etcd-1.etcd-headless.demo-gateway.svc.cluster.local:2379"
            - "http://etcd-2.etcd-headless.demo-gateway.svc.cluster.local:2379"
          prefix: "/apisix"
          timeout: 30
 
      # Admin API 配置
      admin_key:
        - name: admin
          key: xxxxxxxxxxxxxx   # 生产环境用 Secret！
          role: admin
 
      # Prometheus 暴露
      plugin_attr:
        prometheus:
          export_uri: /apisix/prometheus/metrics
          enable_export_server: true
          export_addr:
            ip: 0.0.0.0
            port: 9091
 
    # 日志配置
    nginx_config:
      error_log: "/usr/local/apisix/logs/error.log"
      error_log_level: "warn"
 
      http_configuration_snippet: |
        # 真实客户端 IP——网关前面有 LB/Ingress
        real_ip_header X-Forwarded-For;
        real_ip_recursive on;
        set_real_ip_from 0.0.0.0/0;
 
        # 日志格式——结构化 JSON，方便采集
        log_format main escape=json
          '{'
            '"timestamp":"$time_iso8601",'
            '"remote_addr":"$remote_addr",'
            '"x_forwarded_for":"$http_x_forwarded_for",'
            '"method":"$request_method",'
            '"uri":"$uri",'
            '"status":$status,'
            '"body_bytes_sent":$body_bytes_sent,'
            '"request_time":$request_time,'
            '"upstream_response_time":"$upstream_response_time",'
            '"upstream_addr":"$upstream_addr",'
            '"upstream_status":"$upstream_status",'
            '"http_host":"$http_host",'
            '"http_user_agent":"$http_user_agent",'
            '"route_id":"$http_x_apisix_route_id"'
          '}';
 
        access_log /usr/local/apisix/logs/access.log main buffer=16k flush=3s;

Copy# k8s/gateway-secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: gateway-admin-secret
  namespace: demo-gateway
type: Opaque
stringData:
  admin-key: "实际的生产密钥请用随机字符串生成"

Copy# k8s/gateway-hpa.yaml
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: api-gateway-hpa
  namespace: demo-gateway
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: api-gateway
  minReplicas: 3
  maxReplicas: 20
  behavior:
    # 扩容策略：激进
    scaleUp:
      stabilizationWindowSeconds: 30
      policies:
        - type: Pods
          value: 4           # 每 30s 最多扩容 4 个
          periodSeconds: 30
        - type: Percent
          value: 50          # 或 50%
          periodSeconds: 30
      selectPolicy: Max
    # 缩容策略：保守
    scaleDown:
      stabilizationWindowSeconds: 300   # 等 5 分钟再缩，防止抖动
      policies:
        - type: Pods
          value: 1
          periodSeconds: 60
  metrics:
    # 指标 1：CPU
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 70
 
    # 指标 2：内存
    - type: Resource
      resource:
        name: memory
        target:
          type: Utilization
          averageUtilization: 80
 
    # 指标 3：自定义——QPS（需要 Prometheus Adapter）
    - type: Pods
      pods:
        metric:
          name: apisix_http_requests_per_second
        target:
          type: AverageValue
          averageValue: "150"    # 单 Pod 超过 150 QPS 就扩容

Copyhelm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm install prometheus-adapter prometheus-community/prometheus-adapter \
  --namespace demo-gateway \
  --set prometheus.url=http://prometheus-server.demo-gateway.svc:9090

Copy# 使用 hey 压测
hey -z 120s -c 100 -q 200 \
  http://api-gateway-svc.demo-gateway.svc.cluster.local:9080/demo/echo/hello
 
# 观察扩容
kubectl get hpa -n demo-gateway -w
# 预期: 副本数从 3 → 5 → 8 → ...
 
kubectl get pods -n demo-gateway -w

Copy# ❌ 错误配置
readinessProbe:
  httpGet:
    path: /        # 这个路径可能 404！
    port: 9080
 
# ✅ 正确配置
readinessProbe:
  httpGet:
    path: /apisix/status   # 网关专用健康检查端点
    port: 9080

CopystartupProbe:
  httpGet:
    path: /apisix/status
    port: 9080
  initialDelaySeconds: 10
  periodSeconds: 5
  failureThreshold: 30    # 给足 2.5 分钟启动时间
 
readinessProbe:
  httpGet:
    path: /apisix/status
    port: 9080
  initialDelaySeconds: 5
  periodSeconds: 5
  failureThreshold: 3     # 15 秒没恢复就摘除
 
livenessProbe:
  httpGet:
    path: /apisix/status
    port: 9080
  initialDelaySeconds: 30
  periodSeconds: 15
  failureThreshold: 5     # 75 秒没恢复就重启

Copyresources:
  requests:
    cpu: "500m"
    memory: "512Mi"
  limits:
    cpu: "2000m"
    memory: "2Gi"

Copy# 配置网关日志输出到 stdout/文件
# 在 ConfigMap 中：
nginx_config:
  access_log: "/usr/local/apisix/logs/access.log main buffer=64k flush=5s"
  error_log: "/usr/local/apisix/logs/error.log warn"

Copy# k8s/gateway-deployment.yaml 追加 sidecar
spec:
  containers:
    - name: gateway
      # ... 主容器配置 ...
      volumeMounts:
        - name: shared-logs
          mountPath: /usr/local/apisix/logs
 
    # Sidecar——用 Filebeat/Promtail 采集日志
    - name: log-shipper
      image: grafana/promtail:2.9.0
      args:
        - -config.file=/etc/promtail/promtail.yaml
      volumeMounts:
        - name: shared-logs
          mountPath: /var/log/gateway
        - name: promtail-config
          mountPath: /etc/promtail
      resources:
        requests:
          cpu: 50m
          memory: 64Mi
        limits:
          cpu: 200m
          memory: 128Mi
 
  volumes:
    - name: shared-logs
      emptyDir: {}         # Pod 内共享的日志目录
    - name: promtail-config
      configMap:
        name: promtail-config

Copy# k8s/promtail-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: promtail-config
  namespace: demo-gateway
data:
  promtail.yaml: |
    server:
      http_listen_port: 9080
      grpc_listen_port: 0
 
    clients:
      - url: http://loki.demo-gateway.svc.cluster.local:3100/loki/api/v1/push
 
    positions:
      filename: /tmp/positions.yaml
 
    scrape_configs:
      - job_name: gateway-access-log
        static_configs:
          - targets:
              - localhost
            labels:
              app: api-gateway
              log_type: access
              __path__: /var/log/gateway/access.log
        pipeline_stages:
          - json:
              expressions:
                timestamp: timestamp
                status: status
                request_time: request_time
                upstream_addr: upstream_addr
          - labels:
              status:
              upstream_addr:
 
      - job_name: gateway-error-log
        static_configs:
          - targets:
              - localhost
            labels:
              app: api-gateway
              log_type: error
              __path__: /var/log/gateway/error.log

Copy# k8s/gateway-networkpolicy.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: gateway-network-policy
  namespace: demo-gateway
spec:
  podSelector:
    matchLabels:
      app: api-gateway
  policyTypes:
    - Ingress
    - Egress
  ingress:
    # 允许外部流量访问网关代理端口
    - from:
        - namespaceSelector:
            matchLabels:
              name: ingress-nginx
      ports:
        - protocol: TCP
          port: 9080
        - protocol: TCP
          port: 9443
 
    # 只允许同 namespace 的监控组件访问 Admin API
    - from:
        - podSelector:
            matchLabels:
              app: prometheus
      ports:
        - protocol: TCP
          port: 9180
 
    # 只允许同 namespace 访问 metrics
    - from:
        - podSelector:
            matchLabels:
              app: prometheus
      ports:
        - protocol: TCP
          port: 9091
 
  egress:
    # 网关需要访问 etcd
    - to:
        - podSelector:
            matchLabels:
              app: etcd
      ports:
        - protocol: TCP
          port: 2379
        - protocol: TCP
          port: 2380
 
    # 网关需要访问上游服务（按需放开）
    - to:
        - namespaceSelector: {}
          podSelector: {}
      ports:
        - protocol: TCP
          port: 8080

Copy# 从非允许的 Pod 访问 Admin API（应被拒绝）
kubectl run test-pod --rm -it --image=alpine/curl -- sh
curl http://api-gateway-svc.demo-gateway.svc.cluster.local:9180/admin-api/routes
# 预期: timeout / connection refused

Copy# 在网关 Admin API 中配置金丝雀路由
 
# 1. 创建两个上游：stable 和 canary
curl -s http://127.0.0.1:9180/admin-api/upstreams/demo-svc-stable -X PUT -d '
{
  "type": "roundrobin",
  "nodes": {"demo-svc-stable.demo-gateway.svc.cluster.local:8080": 100}
}'
 
curl -s http://127.0.0.1:9180/admin-api/upstreams/demo-svc-canary -X PUT -d '
{
  "type": "roundrobin",
  "nodes": {"demo-svc-canary.demo-gateway.svc.cluster.local:8080": 100}
}'
 
# 2. 创建两条路由：canary 优先匹配
# Canary 路由——带特定 header
curl -s http://127.0.0.1:9180/admin-api/routes/demo-svc-canary -X PUT -d '
{
  "uri": "/demo/*",
  "name": "demo-svc-canary",
  "priority": 100,
  "vars": [
    ["http_x_canary", "==", "v2"]
  ],
  "upstream_id": "demo-svc-canary",
  "plugins": {
    "traffic-split": {
      "rules": [{
        "weighted_upstreams": [{
          "upstream_id": "demo-svc-canary",
          "weight": 100
        }]
      }]
    }
  }
}'
 
# 主路由——默认
curl -s http://127.0.0.1:9180/admin-api/routes/demo-svc-stable -X PUT -d '
{
  "uri": "/demo/*",
  "name": "demo-svc-stable",
  "priority": 10,
  "upstream_id": "demo-svc-stable"
}'

Copy# 临时：手动压缩 etcd
kubectl exec etcd-0 -- etcdctl compact $(etcdctl endpoint status --write-out=json | jq '.[0].Status.header.revision')
kubectl exec etcd-0 -- etcdctl defrag
 
# 长期：配置自动压缩和配额

Copy# 在 etcd StatefulSet 环境变量中加入
- name: ETCD_AUTO_COMPACTION_RETENTION
  value: "1"          # 保留 1 小时历史
- name: ETCD_QUOTA_BACKEND_BYTES
  value: "8589934592" # 8GB 硬限制

CopyscaleDown:
  stabilizationWindowSeconds: 300   # 等 5 分钟
  policies:
    - type: Percent
      value: 10
      periodSeconds: 60

Copy# 1. 创建 namespace
kubectl create namespace demo-gateway
 
# 2. 部署 etcd 集群
kubectl apply -f k8s/etcd-statefulset.yaml
 
# 3. 部署网关 + 配置
kubectl apply -f k8s/gateway-config.yaml
kubectl apply -f k8s/gateway-secret.yaml
kubectl apply -f k8s/gateway-deployment.yaml
 
# 4. 暴露服务
kubectl apply -f k8s/gateway-service.yaml
 
# 5. 配置 HPA
kubectl apply -f k8s/gateway-hpa.yaml
 
# 6. 配置 Ingress + TLS
kubectl apply -f k8s/cert-issuer.yaml
kubectl apply -f k8s/gateway-certificate.yaml
kubectl apply -f k8s/gateway-ingress.yaml
 
# 7. 部署监控
kubectl apply -f k8s/promtail-config.yaml
kubectl apply -f k8s/gateway-networkpolicy.yaml
 
# 8. 验证
kubectl get all -n demo-gateway
kubectl get hpa -n demo-gateway -w
kubectl describe certificate gateway-tls-cert -n demo-gateway

Copy#!/bin/bash
# k8s/verify.sh
set -e
NS="demo-gateway"
RED='\033[0;31m'
GREEN='\033[0;32m'
NC='\033[0m'
 
check() {
    if eval "$1" > /dev/null 2>&1; then
        echo -e "${GREEN}[PASS]${NC} $2"
    else
        echo -e "${RED}[FAIL]${NC} $2"
    fi
}
 
echo "=== API 网关生产部署验证 ==="
echo ""
 
check "kubectl get ns $NS"                    "Namespace 存在"
check "kubectl get pods -n $NS -l app=etcd --no-headers | grep Running | wc -l | grep 3" \
                                              "etcd 三副本运行中"
check "kubectl exec -n $NS etcd-0 -- etcdctl endpoint health | grep 'is healthy' | wc -l | grep 3" \
                                              "etcd 集群健康"
check "kubectl get deployment api-gateway -n $NS -o jsonpath='{.status.readyReplicas}' | grep -E '^[3-9]|^1[0-9]'" \
                                              "网关 Deployment 运行中 (≥3 副本)"
check "kubectl get hpa api-gateway-hpa -n $NS" \
                                              "HPA 配置生效"
check "kubectl get certificate gateway-tls-cert -n $NS -o jsonpath='{.status.conditions[0].status}' | grep True" \
                                              "TLS 证书已就绪"
check "kubectl get ingress gateway-ingress -n $NS" \
                                              "Ingress 配置正常"
 
echo ""
echo "=== 全部检查完成 ==="