Тестирование Pull Request в Kubernetes с vCluster

metadata:
  labels:
    vcluster.loft.sh/managed-by: vcluster-pipeline-12632713145

kind: NetworkPolicy
apiVersion: networking.k8s.io/v1
metadata:
  name: isolate-vcluster
  namespace: vcluster-vcluster-pipeline-12632713145
spec:
  podSelector: {}
  ingress:
  - from:
    - podSelector: {}

apiVersion: v1
kind: ResourceQuota
metadata:
  name: vcluster-quota
spec:
  hard:
    limits.cpu: "4"
    limits.memory: 8Gi
    requests.cpu: "2"
    requests.memory: 4Gi

syncer:
  extraArgs:
    - --disable-sync-resources=persistentvolumeclaims
storage:
  persistence: false  # SQLite
  # или для полноценного кластера
  # persistence: true  # etcd

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: isolate-vcluster
  namespace: vcluster-pr-1234
spec:
  podSelector: {}
  policyTypes:
  - Ingress
  - Egress
  ingress:
  - from:
    - podSelector: {}
  egress:
  - to:
    - podSelector: {}
    - namespaceSelector:
        matchLabels:
          kubernetes.io/metadata.name: kube-system
    ports:
    - port: 53
      protocol: UDP
    - port: 53
      protocol: TCP

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: app-ingress
spec:
  rules:
  - host: pr-1234-app.example.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: app-service
            port:
              number: 80

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: protect-api-server
  namespace: vcluster-pr-1234
spec:
  podSelector:
    matchLabels:
      app: vcluster
  ingress:
  - from:
    - ipBlock:
        cidr: 10.0.0.0/8

name: Install vCluster
  uses: loft-sh/setup-vcluster@main
  with:
    kubectl-install: false
name: Create a vCluster
  id: vcluster
  run: time vcluster create vcluster-pipeline-${{github.run_id}}
name: Connect to the vCluster
  run: vcluster connect vcluster-pipeline-${{github.run_id}}

name: Install PostgreSQL
  run: |
    helm repo add bitnami [url]https://charts.bitnami.com/bitnami[/url]
    helm install postgres bitnami/postgresql -f postgres-values.yaml
name: Create ConfigMap with DB connection
  run: |
    kubectl create configmap db-config --from-literal=host=postgres-postgresql --from-literal=port=5432
name: Deploy application
  run: |
    kubectl apply -k ./k8s/overlays/test
name: Wait for deployment
  run: |
    kubectl rollout status deployment/my-app

name: Get Service URL
  run: |
    external_ip=""
    while [ -z $external_ip ]; do
      echo "Waiting for external IP..."
      external_ip=$(kubectl get svc my-app -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
      [ -z "$external_ip" ] && sleep 10
    done
    echo "APP_URL=http://$external_ip:8080" >> $GITHUB_ENV

name: Delete the vCluster
  if: ${{ !cancelled() && steps.vcluster.conclusion == 'success' }}
  run: vcluster delete vcluster-pipeline-${{github.run_id}}

# values.yaml для vCluster
syncer:
  extraArgs:
    - --disable-sync-resources=nodes
    - --enforce-pod-security-standard=baseline
storage:
  persistence: false  # Используем SQLite для тестовых окружений
ingress:
  enabled: true
  host: "pr-${PR_NUMBER}.test.example.com"

on:
  issue_comment:
    types: [created]
jobs:
  deploy-test:
    if: github.event.issue.pull_request && contains(github.event.comment.body, '/deploy-test')
    runs-on: ubuntu-latest
    steps:
      # Дальнейшие шаги по созданию vCluster

name: Store cluster info
  if: steps.vcluster.conclusion == 'success'
  run: |
    echo "vcluster-pipeline-${{github.run_id}}" > cluster_name.txt
    aws s3 cp cluster_name.txt s3://my-bucket/pr-${{github.event.pull_request.number}}/

on:
  pull_request:
    types: [closed]
jobs:
  cleanup:
    runs-on: ubuntu-latest
    steps:
      - name: Download cluster info
        run: |
          aws s3 cp s3://my-bucket/pr-${{github.event.pull_request.number}}/cluster_name.txt .
          CLUSTER_NAME=$(cat cluster_name.txt)
          echo "CLUSTER_NAME=$CLUSTER_NAME" >> $GITHUB_ENV
      - name: Install vCluster
        uses: loft-sh/setup-vcluster@main
      - name: Delete vCluster
        run: vcluster delete ${{ env.CLUSTER_NAME }}

name: Set expiration
  run: |
    EXPIRATION=$(date -d "now + 24 hours" +%s)
    echo "$EXPIRATION" > expiration.txt
    aws s3 cp expiration.txt s3://my-bucket/pr-${{github.event.pull_request.number}}/

on:
  schedule:
    - cron: '0 * * * *'  # Каждый час
jobs:
  cleanup-expired:
    runs-on: ubuntu-latest
    steps:
      # Логика проверки и удаления истекших кластеров

variables:
  KUBECONFIG: "$CI_PROJECT_DIR/.kube/config"
 
stages:
  - prepare
  - deploy
  - test
  - cleanup
 
setup_vcluster:
  stage: prepare
  image: alpine:3.14
  script:
    - apk add --no-cache curl
    - curl -L -o vcluster "https://github.com/loft-sh/vcluster/releases/latest/download/vcluster-linux-amd64"
    - chmod +x vcluster
    - mkdir -p .kube
    - ./vcluster create vcluster-$CI_PIPELINE_ID --connect=false
    - ./vcluster connect vcluster-$CI_PIPELINE_ID --update-current=false --kube-config=$KUBECONFIG
  artifacts:
    paths:
      - vcluster
      - .kube/

version: 2.1
jobs:
  deploy:
    docker:
      - image: cimg/base:2022.03
    steps:
      - checkout
      - run:
          name: Install vCluster
          command: |
            curl -L -o vcluster "https://github.com/loft-sh/vcluster/releases/latest/download/vcluster-linux-amd64"
            chmod +x vcluster
      - run:
          name: Create vCluster
          command: |
            mkdir -p $PWD/.kube
            export KUBECONFIG=$PWD/.kube/config
            ./vcluster create vcluster-$CIRCLE_BUILD_NUM

# vcluster-template/Chart.yaml
apiVersion: v2
name: pr-environment
description: PR Test Environment Template
version: 0.1.0

# vcluster-template/values.yaml
vcluster:
  name: "pr-environment"
  namespace: "vcluster-pr"
  persistence: false
  isolation:
    enabled: true
  resources:
    limits:
      cpu: 1
      memory: 1Gi
 
services:
  database:
    enabled: true
    type: postgres
    version: "13"
  
  redis:
    enabled: false

# vcluster-template/templates/vcluster.yaml
apiVersion: v1
kind: Namespace
metadata:
  name: {{ printf "vcluster-%s-%s" .Values.vcluster.name .Release.Name | trunc 63 }}

# pr-1234-values.yaml
vcluster:
  name: "pr-1234"
  resources:
    limits:
      cpu: 2
      memory: 2Gi

apiVersion: v1
kind: ResourceQuota
metadata:
  name: vcluster-quota
spec:
  hard:
    limits.cpu: "4"
    limits.memory: 8Gi
    pods: "20"

apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: high-priority
value: 1000000

apiVersion: v1
kind: LimitRange
metadata:
  name: default-limits
spec:
  limits:
  - default:
      cpu: 500m
      memory: 512Mi

apiVersion: helm.fluxcd.io/v1
kind: HelmRelease
metadata:
name: fluent-bit
spec:
values:
  config:
    outputs: |
      [OUTPUT]
          Name es
          Match *
          Host elasticsearch-master
          Logstash_Format On
          Logstash_Prefix vcluster-${VCLUSTER_NAME}

apiVersion: v1
kind: ConfigMap
metadata:
name: otel-agent-conf
data:
  config.yaml: |
    receivers:
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
    processors:
      batch:
      attributes:
        actions:
          - key: vcluster.name
            value: ${VCLUSTER_NAME}
            action: insert
    exporters:
      otlp:
        endpoint: jaeger-collector:4317

apiVersion: apps/v1
kind: Deployment
metadata:
name: syncer
spec:
template:
  spec:
    containers:
    - name: log-interceptor
      image: busybox
      command: ["/bin/sh", "-c", "tail -f /var/log/syncer/syncer.log | sed 's/^/[vcluster-$VCLUSTER_NAME] /' > /dev/stdout"]
      volumeMounts:
      - name: syncer-logs
        mountPath: /var/log/syncer

apiVersion: batch/v1
kind: Job
metadata:
name: db-init
spec:
template:
spec:
  containers:
  - name: init
    image: my-repo/db-init:latest
    env:
    - name: DB_HOST
      value: postgres
    - name: DB_PASSWORD
      valueFrom:
        secretKeyRef:
          name: db-credentials
          key: postgres-password
  restartPolicy: Never

initContainers:
name: wait-for-api
  image: busybox
  command: ['sh', '-c', 'until nc -z api-service 8080; do echo waiting for api; sleep 2; done;']

apiVersion: batch/v1
kind: CronJob
metadata:
name: cleanup-vclusters
spec:
schedule: "0 * * * *"  # Каждый час
jobTemplate:
spec:
  template:
    spec:
      containers:
      - name: cleanup
        image: my-registry/cleanup-tool:latest
        env:
        - name: GITHUB_TOKEN
          valueFrom:
            secretKeyRef:
              name: github-creds
              key: token
      restartPolicy: OnFailure

metadata:
  labels:
    vcluster.name: "pr-1234"
    team: "backend"
    pr.owner: "username"
    cost-center: "dev-infra"

sum by(vcluster_name) (
  kube_pod_container_resource_requests{resource="cpu"}
  * on(namespace,pod) group_left(vcluster_name)
  kube_pod_labels{label_vcluster_name!=""}
)

apiVersion: batch/v1
kind: CronJob
metadata:
  name: cost-optimizer
spec:
  schedule: "0 * * * *"
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: optimizer
            image: my-repo/cost-optimizer:v1
            args:
            - --idle-threshold=3h
            - --notify-slack=true

resources:
  limits:
    memory: 1Gi
    cpu: 500m
  requests:
    memory: 256Mi
    cpu: 100m

vcluster-pr-testing/
├── .github/
│   └── workflows/
│       └── pr-test.yaml
├── k8s/
│   ├── base/
│   └── overlays/
│       └── test/
└── scripts/
    ├── cleanup.sh
    └── setup-vcluster.sh

name: PR Test
 
on:
  pull_request:
    types: [opened, synchronize, reopened]
 
jobs:
  test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      
      - name: Setup Kubernetes tools
        uses: azure/setup-kubectl@v3
        
      - name: Setup vCluster
        uses: loft-sh/setup-vcluster@main
        
      - name: Create vCluster
        id: create-vcluster
        run: |
          vcluster create pr-${{ github.event.pull_request.number }} \
            --connect=false \
            --distro=k3s \
            --expose
          echo "VCLUSTER_NAME=pr-${{ github.event.pull_request.number }}" >> $GITHUB_ENV
      
      - name: Connect to vCluster
        run: vcluster connect ${{ env.VCLUSTER_NAME }}
      
      - name: Deploy test environment
        run: |
          kubectl apply -k ./k8s/overlays/test
          kubectl rollout status deployment/app
      
      - name: Run tests
        run: ./scripts/run-tests.sh
      
      - name: Delete vCluster
        if: always()
        run: vcluster delete ${{ env.VCLUSTER_NAME }}