Configurre network wit nmstate operator

Kubernetes
Page content

In this post I will show you how you can use nmstate operator to manage your network configurations on a Kubernetes host.

For this Demo I will use a VM with two separate network interface. I will use the first one for the kubernetes network and flannel and use the second one to configure a bridge on ot with nmstate operator.

K3S

curl -sLS https://get.k3sup.dev | sh

mkdir /root/.kube
IP=192.168.100.10

k3sup install --ip $IP --local   --context pik3s \
  --merge \
  --local-path $HOME/.kube/config

k3sup ready --context pik3s

kubernetes-nmstate

yum install NetworkManager -y
systemctl start NetworkManager

dnf copr enable nmstate/nmstate
dnf install nmstate

kubectl apply -f https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.80.1/nmstate.io_nmstates.yaml
kubectl apply -f https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.80.1/namespace.yaml
kubectl apply -f https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.80.1/service_account.yaml
kubectl apply -f https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.80.1/role.yaml
kubectl apply -f https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.80.1/role_binding.yaml
kubectl apply -f https://github.com/nmstate/kubernetes-nmstate/releases/download/v0.80.1/operator.yaml

cat <<EOF | kubectl create -f -
apiVersion: nmstate.io/v1
kind: NMState
metadata:
  name: nmstate
EOF

cat <<EOF | kubectl create -f -
apiVersion: nmstate.io/v1
kind: NodeNetworkConfigurationPolicy
metadata:
  name: br1-enp0s9
spec:
  desiredState:
    interfaces:
    - name: br1
      type: linux-bridge
      state: up
      ipv4:
        address:
        - ip: 192.168.200.10
          prefix-length: 24
	dhcp: false
        enabled: true
      bridge:
	port:
	- name: enp0s9
EOF

multus K3S

wget https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml

nano multus-daemonset.yml
---
...
containers:
  - name: kube-multus
    image: nfvpe/multus:v3.4.1
    command: ["/entrypoint.sh"]
    args:
      - "--multus-conf-file=auto"
      - "--cni-version=0.3.1"
      # Add the following arg
      - "--multus-kubeconfig-file-host=/var/lib/rancher/k3s/agent/etc/cni/net.d/multus.d/multus.kubeconfig"
...
volumes:
  - name: cni
    hostPath:
      path: /var/lib/rancher/k3s/agent/etc/cni/net.d
  - name: cnibin
    hostPath:
      path: /var/lib/rancher/k3s/data/current/bin

nano multus-bridge.yaml
---
apiVersion: k8s.cni.cncf.io/v1
kind: NetworkAttachmentDefinition
metadata:
  name: multus-br1
spec:
  config: |
    {
      "cniVersion": "0.3.1",
      "type": "bridge",
      "bridge": "br1",
      "ipam": {
        "type": "host-local",
        "subnet": "192.168.200.0/24",
         "rangeStart": "192.168.200.240",
         "rangeEnd": "192.168.200.250"
      }
    }

Demo app

nano demo-app.yaml
---
apiVersion: v1
kind: Pod
metadata:
  name: net-pod
  annotations:
    k8s.v1.cni.cncf.io/networks: multus-br1
spec:
  containers:
  - name: netshoot-pod
    image: nicolaka/netshoot
    command: ["tail"]
    args: ["-f", "/dev/null"]
  terminationGracePeriodSeconds: 0
---
apiVersion: v1
kind: Pod
metadata:
  name: net-pod2
  annotations:
    k8s.v1.cni.cncf.io/networks: multus-br1
spec:
  containers:
  - name: netshoot-pod
    image: nicolaka/netshoot
    command: ["tail"]
    args: ["-f", "/dev/null"]
  terminationGracePeriodSeconds: 0

kubectl exec -it net-pod -- ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0@if22: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
    link/ether 56:60:ac:ef:12:be brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 10.42.0.14/24 brd 10.42.0.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::5460:acff:feef:12be/64 scope link
       valid_lft forever preferred_lft forever
3: net1@if24: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether a2:0a:ec:07:64:6b brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.200.240/24 brd 192.168.200.255 scope global net1
       valid_lft forever preferred_lft forever
    inet6 fe80::a00a:ecff:fe07:646b/64 scope link
       valid_lft forever preferred_lft forever

kubectl exec -it net-pod2 -- ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0@if23: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UP group default
    link/ether 2e:b3:6e:d8:cf:35 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 10.42.0.15/24 brd 10.42.0.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::2cb3:6eff:fed8:cf35/64 scope link
       valid_lft forever preferred_lft forever
3: net1@if25: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 52:96:53:1a:58:15 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.200.241/24 brd 192.168.200.255 scope global net1
       valid_lft forever preferred_lft forever
    inet6 fe80::5096:53ff:fe1a:5815/64 scope link
       valid_lft forever preferred_lft forever

# ping own ip
kubectl exec -it net-pod -- ping -c 1 -I net1 192.168.200.241
PING 192.168.200.241 (192.168.200.241) from 192.168.200.240 net1: 56(84) bytes of data.
64 bytes from 192.168.200.241: icmp_seq=1 ttl=64 time=0.220 ms

--- 192.168.200.241 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.220/0.220/0.220/0.000 ms

# ping net-pod2's ip
kubectl exec -it net-pod2 -- ping -c 1 -I net1 192.168.200.240
PING 192.168.200.240 (192.168.200.240) from 192.168.200.241 net1: 56(84) bytes of data.
64 bytes from 192.168.200.240: icmp_seq=1 ttl=64 time=0.072 ms

--- 192.168.200.240 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.072/0.072/0.072/0.000 ms

# ping host
kubectl exec -it net-pod -- ping -c 1 -I net1 192.168.200.10
PING 192.168.200.10 (192.168.200.10) from 192.168.200.240 net1: 56(84) bytes of data.
64 bytes from 192.168.200.10: icmp_seq=1 ttl=64 time=0.082 ms

--- 192.168.200.10 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.082/0.082/0.082/0.000 ms