Kubernetes Egress Gateway with Antrea

# Check Antrea version
kubectl get pods -n kube-system -l app=antrea -o jsonpath='{.items[0].spec.containers[0].image}'

# Verify Antrea components
kubectl get pods -n kube-system -l app=antrea

# Expected output:
# NAME                    READY   STATUS
# antrea-agent-xxxxx      2/2     Running
# antrea-controller-xxxxx 1/1     Running

# Check Antrea feature gates
kubectl get configmap antrea-config -n kube-system -o yaml | \
  grep -A10 featureGates

# View Antrea configuration
kubectl get configmap antrea-config -n kube-system -o yaml

# Using Helm (recommended for new installations)
helm upgrade antrea antrea/antrea \
  -n kube-system \
  --set featureGates.Egress=true \
  --set featureGates.ExternalNode=true \
  --wait

# Edit Antrea ConfigMap
kubectl edit configmap antrea-config -n kube-system

# Add/modify featureGates:
apiVersion: v1
kind: ConfigMap
metadata:
  name: antrea-config
  namespace: kube-system
data:
  antrea.yaml: |
    featureGates:
      Egress: true
      ExternalNode: true

# Restart Antrea pods to apply changes
kubectl rollout restart daemonset antrea-agent -n kube-system
kubectl rollout restart deployment antrea-controller -n kube-system

# Wait for pods to be ready
kubectl wait --for=condition=Ready pods -l app=antrea -n kube-system --timeout=2m

# Verify Egress is enabled
kubectl exec -n kube-system antrea-agent-xxxxx -- antctl get featuregates | grep Egress

apiVersion: crd.antrea.io/v1beta1
kind: ExternalIPPool
metadata:
  name: egress-pool-1
spec:
  # Range of IPs for egress NAT
  ipRanges:
    - start: 192.168.100.10
      end: 192.168.100.20
  
  # Nodes that can serve as egress gateways
  nodeSelector:
    matchLabels:
      node-role.kubernetes.io/egress-gateway: ""
  
  # Optional: Auto-assign IPs from pool
  autoAssign: true

kubectl apply -f external-ippool.yaml

# List external IP pools
kubectl get externalippool

# Get detailed status
kubectl describe externalippool egress-pool-1

# Check allocated IPs
kubectl get externalippool egress-pool-1 -o yaml

apiVersion: crd.antrea.io/v1beta1
kind: Egress
metadata:
  name: egress-production
spec:
  # Reference the IP pool
  externalIPPool: egress-pool-1
  
  # Select pods/namespaces that use this egress
  appliedTo:
    - namespaceSelector:
        matchLabels:
          name: production
      podSelector:
        matchLabels:
          app: api-server
  
  # Optional: Specific external destinations
  # If not specified, all external traffic uses egress
  destination:
    - ipBlock:
        cidr: 0.0.0.0/0
        # Exclude internal networks
        except:
          - 10.0.0.0/8
          - 172.16.0.0/12
          - 192.168.0.0/16

kubectl apply -f egress.yaml

# List egress resources
kubectl get egress

# Get detailed status
kubectl describe egress egress-production

# Check which gateway node is assigned
kubectl get egress egress-production -o jsonpath='{.status.egressIP}'
kubectl get egress egress-production -o jsonpath='{.status.gatewayNode}'

# Label a node as egress gateway
kubectl label node worker-node-2 \
  node-role.kubernetes.io/egress-gateway=

# Verify label
kubectl get nodes -l node-role.kubernetes.io/egress-gateway=

# Label multiple gateway nodes
kubectl label node worker-node-2 \
  node-role.kubernetes.io/egress-gateway=
kubectl label node worker-node-3 \
  node-role.kubernetes.io/egress-gateway=

# Antrea will automatically failover if primary gateway fails

# Create test namespace
kubectl create namespace test-egress
kubectl label namespace test-egress name=test-egress

# Deploy test pod
kubectl run test-pod --image=curlimages/curl -it --rm --restart=Never \
  --namespace test-egress \
  --labels app=test-app \
  -- curl -s ifconfig.me

apiVersion: crd.antrea.io/v1beta1
kind: Egress
metadata:
  name: egress-test
spec:
  externalIPPool: egress-pool-1
  appliedTo:
    - namespaceSelector:
        matchLabels:
          name: test-egress
      podSelector:
        matchLabels:
          app: test-app

# From pod with egress
kubectl exec -it test-pod -n test-egress -- curl -s ifconfig.me
# Should show egress IP from pool (e.g., 192.168.100.10)

# From pod without egress
kubectl run normal-pod --image=curlimages/curl -it --rm --restart=Never \
  -- curl -s ifconfig.me
# Should show node IP (not from egress pool)

# Production egress pool
apiVersion: crd.antrea.io/v1beta1
kind: ExternalIPPool
metadata:
  name: egress-pool-production
spec:
  ipRanges:
    - start: 192.168.100.10
      end: 192.168.100.20
  nodeSelector:
    matchLabels:
      node-role.kubernetes.io/egress-gateway: ""

---
# Staging egress pool
apiVersion: crd.antrea.io/v1beta1
kind: ExternalIPPool
metadata:
  name: egress-pool-staging
spec:
  ipRanges:
    - start: 192.168.100.30
      end: 192.168.100.40
  nodeSelector:
    matchLabels:
      node-role.kubernetes.io/egress-gateway: ""

apiVersion: crd.antrea.io/v1beta1
kind: Egress
metadata:
  name: egress-namespace-wide
spec:
  externalIPPool: egress-pool-production
  appliedTo:
    - namespaceSelector:
        matchLabels:
          name: production
  # No podSelector = all pods in namespace

apiVersion: crd.antrea.io/v1alpha1
kind: ExternalNode
metadata:
  name: vm-server-1
  namespace: external
spec:
  interfaces:
    - interfaceName: eth0
      ips:
        - 192.168.50.10/24
  nodeAnnotations:
    type: "vm"

apiVersion: crd.antrea.io/v1beta1
kind: Egress
metadata:
  name: egress-ha
spec:
  externalIPPool: egress-pool-1
  appliedTo:
    - namespaceSelector:
        matchLabels:
          name: production
  
  # Antrea automatically handles failover
  # When primary gateway fails, secondary takes over
  # No additional configuration needed

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: restrict-egress
  namespace: production
spec:
  podSelector:
    matchLabels:
      app: api-server
  policyTypes:
    - Egress
  egress:
    # Allow DNS
    - to:
        - namespaceSelector: {}
          podSelector:
            matchLabels:
              k8s-app: kube-dns
      ports:
        - protocol: UDP
          port: 53
    
    # Allow only specific external services
    - to:
        - ipBlock:
            cidr: 52.0.0.0/8  # AWS
      ports:
        - protocol: TCP
          port: 443
    
    # All other traffic goes through egress gateway
    - to:
        - ipBlock:
            cidr: 0.0.0.0/0
            except:
              - 10.0.0.0/8
              - 172.16.0.0/12
              - 192.168.0.0/16

apiVersion: crd.antrea.io/v1beta1
kind: ClusterNetworkPolicy
metadata:
  name: egress-control
spec:
  priority: 10
  appliedTo:
    - podSelector:
        matchLabels:
          app: api-server
  egress:
    - action: Allow
      to:
        - ipBlock:
            cidr: 52.0.0.0/8  # AWS only
      protocols:
        - protocol: TCP
          port: 443
    - action: Drop
      to:
        - ipBlock:
            cidr: 0.0.0.0/0

# Enable metrics in Antrea config
kubectl edit configmap antrea-config -n kube-system

# Add:
# flowExporter:
#   enable: true
#   flowCollectorAddr: "flow-collector:4739"

# Access Prometheus metrics
kubectl port-forward -n kube-system svc/antrea-metrics 9090:9090

apiVersion: v1
kind: ConfigMap
metadata:
  name: antrea-egress-dashboard
  namespace: monitoring
data:
  antrea-egress.json: |
    {
      "dashboard": {
        "title": "Antrea Egress Gateway",
        "panels": [
          {
            "title": "Egress IP Usage",
            "targets": [
              {
                "expr": "antrea_egress_ip_allocations"
              }
            ]
          },
          {
            "title": "Egress Traffic",
            "targets": [
              {
                "expr": "rate(antrea_egress_gateway_bytes[5m])"
              }
            ]
          }
        ]
      }
    }

# Enable flow exporter for detailed traffic analysis
helm upgrade antrea antrea/antrea \
  -n kube-system \
  --set featureGates.FlowExporter=true \
  --set flowExporter.enable=true \
  --set flowExporter.flowCollectorAddr="flow-collector.monitoring:4739" \
  --wait

# Check ExternalIPPool status
kubectl get externalippool

# Verify node selector matches
kubectl get nodes --show-labels | grep egress-gateway

# Check Egress resource status
kubectl describe egress egress-production

# Look for events
kubectl get events -n kube-system | grep -i egress

# Verify pod labels match Egress spec
kubectl get pods -n production --show-labels

# Check if Egress is applied
kubectl get egress -A

# Test from inside pod
kubectl exec -it test-pod -n production -- \
  curl -s ifconfig.me

# Check OVS flows on gateway node
kubectl debug node/gateway-node -it --image=antrea/antrea-agent-ubuntu -- \
  ovs-ofctl dump-flows br-int

# Check gateway node status
kubectl get nodes

# View egress status after failover
kubectl get egress egress-production -o yaml

# Check which node is now handling egress
kubectl get egress egress-production -o jsonpath='{.status.gatewayNode}'

# View Antrea agent logs
kubectl logs -n kube-system -l app=antrea-agent | grep -i egress

# Check OVS flow statistics
kubectl debug node/gateway-node -it --image=antrea/antrea-agent-ubuntu -- \
  ovs-ofctl dump-flows br-int table=0

# Check Antrea agent metrics
kubectl port-forward -n kube-system svc/antrea-agent 9091:9091
curl localhost:9091/metrics | grep egress

apiVersion: crd.antrea.io/v1beta1
kind: Egress
metadata:
  name: restricted-egress
spec:
  externalIPPool: egress-pool-1
  appliedTo:
    - namespaceSelector:
        matchLabels:
          name: production
  destination:
    - ipBlock:
        cidr: 52.0.0.0/8      # AWS only
    - ipBlock:
        cidr: 35.0.0.0/8      # GCP only

# Production - dedicated IPs
apiVersion: crd.antrea.io/v1beta1
kind: ExternalIPPool
metadata:
  name: egress-production
spec:
  ipRanges:
    - start: 192.168.100.10
      end: 192.168.100.20

---
# Staging - separate IPs
apiVersion: crd.antrea.io/v1beta1
kind: ExternalIPPool
metadata:
  name: egress-staging
spec:
  ipRanges:
    - start: 192.168.100.30
      end: 192.168.100.40

# Enable flow logging for audit
apiVersion: crd.antrea.io/v1alpha1
kind: Traceflow
metadata:
  name: egress-trace
spec:
  source:
    namespace: production
    pod: api-server-xxxxx
  destination:
    ip: "8.8.8.8"
  packet:
    ipHeader:
      protocol: 6  # TCP
      ttl: 128

# 1. Install Antrea alongside Calico (dual-stack temporarily)
helm install antrea antrea/antrea \
  -n kube-system \
  --create-namespace

# 2. Configure egress in Antrea
kubectl apply -f external-ippool.yaml
kubectl apply -f egress.yaml

# 3. Test egress functionality
kubectl run test --image=curlimages/curl -- curl ifconfig.me

# 4. Remove Calico (after validation)
kubectl delete -f calico.yaml

# Note: Requires cluster recreation or careful migration
# 1. Backup all resources
kubectl get all -A -o yaml > backup.yaml

# 2. Install Antrea
helm install antrea antrea/antrea -n kube-system

# 3. Remove Flannel
kubectl delete -f flannel.yaml

# 4. Restore workloads
kubectl apply -f backup.yaml