49 Wartung und Updates

# Verfügbare Update-Channels anzeigen
oc get clusterversion version -o yaml | grep availableUpdates -A 20

# Update-Channel wechseln
oc patch clusterversion version --type merge -p '{"spec":{"channel":"stable-4.13"}}'

# Update-Status überprüfen
oc get clusterversion
oc describe clusterversion version

# Update-Progress monitoren
oc get clusteroperators

# CVE-Status für Images prüfen
oc get images -o json | jq '.items[] | select(.dockerImageMetadata.Config.Labels."io.openshift.build.commit.id")'

# Security-Scanning-Ergebnisse anzeigen
oc describe images sha256:abcd1234... | grep -A 10 "vulnerabilities"

# Control Plane Update initiieren
oc adm upgrade --to=4.13.12

# Update-Fortschritt überwachen
watch -n 5 "oc get clusteroperators | grep -v 'True.*False.*False'"

# Update-History anzeigen
oc get clusterversion version -o yaml | grep -A 20 history

# Node für Update vorbereiten
oc adm drain worker-node-1 --ignore-daemonsets --delete-emptydir-data

# Node-Update-Status prüfen
oc get nodes -l node.openshift.io/os_id=rhcos -o wide

# Node nach Update wieder aktivieren
oc adm uncordon worker-node-1

# Operator-Updates anzeigen
oc get csv -A | grep -v Succeeded

# Spezifischen Operator aktualisieren
oc patch subscription my-operator -n my-namespace --type merge \
  -p '{"spec":{"channel":"stable","installPlanApproval":"Automatic"}}'

# Anwendungskompatibilität testen
oc get deployments -A -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.template.spec.containers[0].image}{"\n"}{end}'

# Custom Resources auf Kompatibilität prüfen
oc get crd -o custom-columns=NAME:.metadata.name,VERSION:.spec.versions[0].name

# etcd-Backup erstellen
oc get no -l node-role.kubernetes.io/master -o jsonpath='{.items[0].metadata.name}' | \
xargs -I {} oc debug node/{} -- chroot /host sudo -E /usr/local/bin/cluster-backup.sh /home/core/backup

# Backup-Integrität verifizieren
oc debug node/master-0 -- chroot /host \
  sudo ETCDCTL_API=3 etcdctl --cert /etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-metrics-master-0.crt \
  --key /etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-serving-metrics-master-0.key \
  --cacert /etc/kubernetes/static-pod-certs/configmaps/etcd-serving-ca/ca-bundle.crt \
  --endpoints=https://localhost:2379 snapshot status /home/core/backup/snapshot_*.db

# Performance-Baseline sammeln
oc adm top nodes > baseline-nodes-$(date +%Y%m%d).txt
oc adm top pods -A > baseline-pods-$(date +%Y%m%d).txt

# Cluster-Metriken vor Update
oc exec -n openshift-monitoring prometheus-k8s-0 -- \
  promtool query instant 'cluster:cpu_usage_cores:sum' > cpu-baseline.txt

# Automated Health-Check-Script
#!/bin/bash
echo "=== Cluster Health Check ==="

# Cluster Operators Status
echo "Cluster Operators:"
oc get clusteroperators | grep -v "True.*False.*False" | head -5

# Node Status
echo -e "\nNode Status:"
oc get nodes | grep -v Ready | head -5

# Pod Status
echo -e "\nProblem Pods:"
oc get pods -A | grep -v Running | grep -v Completed | head -10

# API Server Responsiveness
echo -e "\nAPI Response Time:"
time oc get namespaces > /dev/null

# Performance-Vergleich nach Update
echo "=== Performance Comparison ==="

# CPU-Nutzung vergleichen
echo "CPU Usage Comparison:"
echo "Before: $(cat cpu-baseline.txt)"
echo "After: $(oc exec -n openshift-monitoring prometheus-k8s-0 -- promtool query instant 'cluster:cpu_usage_cores:sum')"

# Memory-Nutzung vergleichen
echo "Memory Usage Comparison:"
oc adm top nodes --no-headers | awk '{sum+=$5} END {print "Current Memory Usage: " sum "%"}'

# Smoke-Test-Pod
apiVersion: v1
kind: Pod
metadata:
  name: post-update-smoke-test
spec:
  restartPolicy: Never
  containers:
  - name: test
    image: curlimages/curl:latest
    command:
    - /bin/sh
    - -c
    - |
      set -e
      echo "Testing internal services..."
      curl -f http://my-service:8080/health
      echo "Testing external connectivity..."
      curl -f https://registry.redhat.io/v2/
      echo "All tests passed!"

# Rollback-Monitoring-Script
#!/bin/bash
HEALTH_THRESHOLD=80
FAILED_OPERATORS=$(oc get co | grep -v "True.*False.*False" | wc -l)

if [ $FAILED_OPERATORS -gt 3 ]; then
    echo "Critical: $FAILED_OPERATORS operators failing. Initiating rollback..."
    oc adm upgrade --to-latest=false --allow-explicit-channel
    
    # Benachrichtigung senden
    curl -X POST -H 'Content-type: application/json' \
      --data '{"text":"OpenShift rollback initiated due to operator failures"}' \
      $SLACK_WEBHOOK_URL
fi

# etcd-Snapshot-Recovery
# 1. Cluster herunterfahren
oc patch etcd cluster -p='{"spec":{"managementState":"Unmanaged"}}' --type=merge

# 2. Snapshot wiederherstellen
oc debug node/master-0 -- chroot /host \
  sudo /usr/local/bin/cluster-restore.sh /home/core/backup/snapshot_2024010112345.db

# 3. Cluster neu starten
oc patch etcd cluster -p='{"spec":{"managementState":"Managed"}}' --type=merge

# Automatisiertes Cleanup-Script
#!/bin/bash
echo "=== OpenShift Cleanup Automation ==="

# Beendete Pods löschen
oc delete pods --field-selector=status.phase=Succeeded -A

# Nicht verwendete Images bereinigen
oc adm prune images --keep-tag-revisions=3 --keep-younger-than=72h

# Build-Artefakte bereinigen
oc adm prune builds --orphans --keep-complete=5 --keep-failed=2

# Deployments bereinigen
oc adm prune deployments --keep-complete=5 --keep-failed=2

echo "Cleanup completed at $(date)"

# Zertifikat-Status überwachen
oc get secrets -A -o json | jq -r '
  .items[] | 
  select(.type=="kubernetes.io/tls") | 
  "\(.metadata.namespace)/\(.metadata.name): \(.data."tls.crt" // empty)"
' | while read cert; do
  if [ -n "$cert" ]; then
    echo "$cert" | base64 -d | openssl x509 -noout -dates
  fi
done

# Log-Forwarding-Konfiguration
apiVersion: logging.openshift.io/v1
kind: ClusterLogForwarder
metadata:
  name: instance
spec:
  outputs:
  - name: remote-loki
    type: loki
    url: https://loki.company.com:3100
    loki:
      tenantKey: kubernetes.namespace_name
  pipelines:
  - name: container-logs
    inputRefs:
    - application
    outputRefs:
    - remote-loki
    filterRefs:
    - json-filter

# Kapazitäts-Trend-Analyse
#!/bin/bash
echo "=== Capacity Trend Analysis ==="

# 30-Tage-CPU-Trend
oc exec -n openshift-monitoring prometheus-k8s-0 -- \
  promtool query instant 'predict_linear(cluster:cpu_usage_cores:sum[30d], 86400 * 30)'

# 30-Tage-Memory-Trend  
oc exec -n openshift-monitoring prometheus-k8s-0 -- \
  promtool query instant 'predict_linear(cluster:memory_usage:sum[30d], 86400 * 30)'

# Storage-Wachstum
oc exec -n openshift-monitoring prometheus-k8s-0 -- \
  promtool query instant 'predict_linear(cluster:storage_usage:sum[30d], 86400 * 30)'

# Kapazitäts-Alert-Regeln
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: capacity-alerts
spec:
  groups:
  - name: capacity.rules
    rules:
    - alert: ClusterCPUCapacityHigh
      expr: cluster:cpu_usage_cores:sum / cluster:cpu_capacity_cores:sum > 0.8
      for: 15m
      labels:
        severity: warning
      annotations:
        summary: "Cluster CPU capacity is running high"
        
    - alert: ClusterMemoryCapacityHigh
      expr: cluster:memory_usage:sum / cluster:memory_capacity:sum > 0.85
      for: 10m
      labels:
        severity: warning
      annotations:
        summary: "Cluster memory capacity is running high"

{
  "dashboard": {
    "title": "OpenShift Platform Health",
    "panels": [
      {
        "title": "Cluster Operators Status",
        "type": "table",
        "targets": [
          {
            "expr": "cluster_operator_conditions{condition=\"Available\"}",
            "format": "table"
          }
        ]
      },
      {
        "title": "Node Resource Utilization",
        "type": "graph",
        "targets": [
          {
            "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])))",
            "legendFormat": "CPU Usage %"
          }
        ]
      }
    ]
  }
}

# Predictive Maintenance ConfigMap
apiVersion: v1
kind: ConfigMap
metadata:
  name: predictive-maintenance-config
data:
  config.yaml: |
    metrics:
      - name: node_failure_prediction
        query: increase(node_boot_time_seconds[1h])
        threshold: 0.1
        action: alert
      - name: disk_failure_prediction
        query: rate(node_disk_io_time_seconds_total[5m])
        threshold: 0.95
        action: maintenance_schedule

# Single-Node-Wartungs-Script
#!/bin/bash
echo "=== Single-Node Maintenance ==="

# Resource-schonende Updates
oc patch clusterversion version --type merge -p '{"spec":{"desiredUpdate":{"force":false}}}'

# Memory-bewusstes Cleanup
oc delete pods --field-selector=status.phase=Succeeded -A
oc adm prune images --keep-tag-revisions=2 --keep-younger-than=24h

# CodeReady Containers spezifisch
if command -v crc &> /dev/null; then
    crc stop
    crc cleanup
    crc start
fi

# Vereinfachtes CRC-Update
#!/bin/bash
echo "Updating CodeReady Containers..."

# Backup der aktuellen Konfiguration
crc config view > crc-config-backup.txt

# Update durchführen
crc stop
crc delete -f
# Neue CRC-Version installieren
crc setup
crc start

echo "Update completed. Restore projects manually if needed."

# Audit-Logging-Konfiguration
apiVersion: config.openshift.io/v1
kind: APIServer
metadata:
  name: cluster
spec:
  audit:
    profile: WriteRequestBodies
    customRules:
    - level: Request
      resources:
      - group: ""
        resources: ["pods", "services"]
      - group: "apps"
        resources: ["deployments", "replicasets"]

# GitOps-basiertes Change Management
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: cluster-config
spec:
  source:
    repoURL: https://git.company.com/openshift-config
    path: clusters/production
    targetRevision: main
  destination:
    server: https://kubernetes.default.svc
  syncPolicy:
    automated:
      prune: false
      selfHeal: false
    syncOptions:
    - CreateNamespace=false
    - ApplyOutOfSyncOnly=true

# Wartungsfenster-Planer
#!/bin/bash
# Beste Wartungszeiten basierend auf historischer Last ermitteln
oc exec -n openshift-monitoring prometheus-k8s-0 -- \
  promtool query instant 'avg_over_time(cluster:cpu_usage_cores:sum[7d])' | \
  awk 'BEGIN{min=100} {if($2<min && $2>0){min=$2; time=$1}} END{print "Optimal maintenance window: " time}'

# PodDisruptionBudget für kritische Services
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: critical-app-pdb
spec:
  minAvailable: 2
  selector:
    matchLabels:
      app: critical-app
      tier: frontend