Автомасштабирование Kubernetes: HPA, VPA и автомасштабирование кластера

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: my-api
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: my-api
  minReplicas: 2
  maxReplicas: 10
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 75

metrics:
type: Resource
  resource:
    name: cpu
    target:
      type: Utilization
      averageUtilization: 80
type: Pods
  pods:
    metric:
      name: packets-per-second
    target:
      type: AverageValue
      averageValue: 1k

desiredReplicas = ceil[currentReplicas * (currentMetricValue / desiredMetricValue)]

# Пример конфликта, когда VPA пытается изменить ресурсы пода, защищенного PDB
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: api-pdb
spec:
minAvailable: 90%
selector:
  matchLabels:
    app: critical-api

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: webapp-hpa
spec:
scaleTargetRef:
  apiVersion: apps/v1
  kind: Deployment
  name: webapp
minReplicas: 3
maxReplicas: 15
metrics:
type: Resource
  resource:
    name: cpu
    target:
      type: Utilization
      averageUtilization: 70

metrics:
type: Resource
resource:
  name: memory
  target:
    type: AverageValue
    averageValue: 250Mi

metrics:
type: Resource
resource:
  name: cpu
  target:
    type: Utilization
    averageUtilization: 70
type: Resource
resource:
  name: memory
  target:
    type: Utilization
    averageUtilization: 80
type: Pods
pods:
  metric:
    name: packets-per-second
  target:
    type: AverageValue
    averageValue: 1k

metrics:
type: External
external:
  metric:
    name: kafka_consumergroup_lag
    selector:
      matchLabels:
        topic: orders
  target:
    type: AverageValue
    averageValue: 100

metrics:
type: Object
  object:
    describedObject:
      apiVersion: networking.k8s.io/v1
      kind: Ingress
      name: main-ingress
    metric:
      name: requests-per-second
    target:
      type: Value
      value: 10k

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: stable-hpa
spec:
  # ... стандартные поля ...
  behavior:
    scaleDown:
      stabilizationWindowSeconds: 300
      policies:
      - type: Percent
        value: 50
        periodSeconds: 60
    scaleUp:
      stabilizationWindowSeconds: 60
      policies:
      - type: Percent
        value: 100
        periodSeconds: 60
      - type: Pods
        value: 4
        periodSeconds: 60
      selectPolicy: Max

prometheus:
  url: [url]http://prometheus-server.monitoring.svc.cluster.local[/url]
  port: 9090
 
rules:
  default: false
  custom:
  - seriesQuery: 'http_requests_total{namespace!="",pod!=""}'
    resources:
      overrides:
        namespace:
          resource: namespace
        pod:
          resource: pod
    name:
      matches: "^(.*)_total"
      as: "${1}_per_second"
    metricsQuery: 'sum(rate(<<.Series>>{<<.LabelMatchers>>}[2m])) by (<<.GroupBy>>)'

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: api-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: api
  minReplicas: 2
  maxReplicas: 20
  metrics:
  - type: Pods
    pods:
      metric:
        name: http_requests_per_second
      target:
        type: AverageValue
        averageValue: 100

seriesQuery: 'http_request_duration_seconds_sum{namespace!="",pod!=""}'
  resources:
    overrides:
      namespace: {resource: "namespace"}
      pod: {resource: "pod"}
  name:
    matches: "http_request_duration_seconds_sum"
    as: "http_request_latency"
  metricsQuery: 'sum(rate(http_request_duration_seconds_sum{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) / sum(rate(http_request_duration_seconds_count{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>)'

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: sqs-consumer-hpa
spec:
scaleTargetRef:
  apiVersion: apps/v1
  kind: Deployment
  name: sqs-consumer
minReplicas: 1
maxReplicas: 10
metrics:
type: External
  external:
    metric:
      name: sqs-approximage-message-count
      selector:
        matchLabels:
          queue: orders-queue
    target:
      type: AverageValue
      averageValue: 5

behavior:
  scaleDown:
    stabilizationWindowSeconds: 300  # Думай 5 минут перед сокращением
  scaleUp:
    stabilizationWindowSeconds: 60   # Думай минуту перед ростом

behavior:
  scaleDown:
    policies:
    - type: Percent
      value: 20
      periodSeconds: 60  # Не более 20% подов за минуту
  scaleUp:
    policies:
    - type: Pods
      value: 5
      periodSeconds: 60  # Не более 5 подов за минуту
    - type: Percent
      value: 200
      periodSeconds: 60  # Не более чем удвоение за минуту
    selectPolicy: Max    # Выбирать более агрессивную политику

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: multi-container-app
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: complex-app
  minReplicas: 2
  maxReplicas: 10
  metrics:
  - type: ContainerResource
    containerResource:
      name: cpu
      container: analytics-engine
      target:
        type: Utilization
        averageUtilization: 70

metrics:
type: ContainerResource
  containerResource:
    name: cpu
    container: api-server
    target:
      type: Utilization
      averageUtilization: 75
type: ContainerResource
  containerResource:
    name: memory
    container: cache
    target:
      type: Utilization
      averageUtilization: 80

apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
  name: mysql-vpa
spec:
  targetRef:
    apiVersion: apps/v1
    kind: StatefulSet
    name: mysql
  updatePolicy:
    updateMode: "Auto"
  resourcePolicy:
    containerPolicies:
    - containerName: '*'
      minAllowed:
        cpu: 100m
        memory: 512Mi
      maxAllowed:
        cpu: 4
        memory: 8Gi

updatePolicy:
updateMode: "Auto"

updatePolicy:
updateMode: "Initial"

updatePolicy:
updateMode: "Off"

updatePolicy:
updateMode: "Recreate"

apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
  name: analytics-vpa
spec:
  targetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: analytics-engine
  updatePolicy:
    updateMode: "Off"  # Только рекомендации

Status:
  Recommendation:
    Container Recommendations:
      Container Name:  analytics-engine
      Lower Bound:
        Cpu:     500m
        Memory:  512Mi
      Target:
        Cpu:     750m
        Memory:  1Gi
      Upper Bound:
        Cpu:     1
        Memory:  1.5Gi

containerPolicies:
containerName: '*'
  controlledResources: ["cpu", "memory"]
  mode: Auto
  minAllowed:
    cpu: 100m
    memory: 250Mi

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: cluster-autoscaler-priority-expander
  namespace: kube-system
data:
  priorities: |-
    10:
      - .*spot.*
    50:
      - .*

--scale-down-utilization-threshold=0.5

--scale-down-delay-after-add=10m

apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: critical-api-pdb
spec:
  minAvailable: 2
  selector:
    matchLabels:
      app: critical-api

apiVersion: apps/v1
kind: Deployment
metadata:
  name: ml-inference
spec:
  template:
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: gpu
                operator: Exists

kubectl taint nodes gpu-node-1 workload=gpu:NoSchedule

tolerations:
key: "workload"
  operator: "Equal"
  value: "gpu"
  effect: "NoSchedule"

# HPA с более длинным окном для сглаживания скачков
behavior:
  scaleUp:
    stabilizationWindowSeconds: 120
  scaleDown:
    stabilizationWindowSeconds: 300

--scale-up-unneeded-time=10m
--ok-total-unready-count=3
--max-node-provision-time=15m

# HPA по кастомной метрике
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: service-hpa
spec:
  metrics:
  - type: External
    external:
      metric:
        name: nginx_connections_active
      target:
        type: AverageValue
        averageValue: 800

affinity:
  podAntiAffinity:
    preferredDuringSchedulingIgnoredDuringExecution:
    - weight: 100
      podAffinityTerm:
        labelSelector:
          matchExpressions:
          - key: app
            operator: In
            values:
            - critical-api
        topologyKey: "kubernetes.io/hostname"

topologySpreadConstraints:
maxSkew: 1
  topologyKey: topology.kubernetes.io/zone
  whenUnsatisfiable: ScheduleAnyway
  labelSelector:
    matchLabels:
      app: critical-service

apiVersion: batch/v1
kind: CronJob
metadata:
name: scale-up-morning
spec:
schedule: "30 8 * * 1-5"  # В 8:30 по будням
jobTemplate:
  spec:
    template:
      spec:
        containers:
        - name: kubectl
          image: bitnami/kubectl
          command:
          - kubectl
          - scale
          - deployment/web-app
          - --replicas=10
        restartPolicy: OnFailure

apiVersion: apps/v1
kind: Deployment
metadata:
  name: cluster-autoscaler
  namespace: kube-system
spec:
  containers:
  - command:
    - ./cluster-autoscaler
    - --cloud-provider=aws
    - --node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/my-cluster
    env:
    - name: AWS_REGION
      value: us-east-1

nodeSelector:
  cloud: aws