Flannel IPSec 模式
一、环境信息
主机 | IP |
---|---|
ubuntu | 172.16.94.141 |
软件 | 版本 |
---|---|
docker | 26.1.4 |
helm | v3.15.0-rc.2 |
kind | 0.18.0 |
clab | 0.54.2 |
kubernetes | 1.23.4 |
ubuntu os | Ubuntu 20.04.6 LTS |
kernel | 5.11.5 内核升级文档 |
二、安装服务
kind
配置文件信息
$ cat install.sh#!/bin/bash
date
set -v# 1.prep noCNI env
cat <<EOF | kind create cluster --name=flannel-ipsec --image=kindest/node:v1.23.4 --config=-
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
networking:# kind 默认使用 rancher cni,cni 我们需要自己创建disableDefaultCNI: true# 定义节点使用的 pod 网段podSubnet: "10.244.0.0/16"
nodes:- role: control-plane- role: worker- role: workercontainerdConfigPatches:
- |-[plugins."io.containerd.grpc.v1.cri".registry.mirrors."harbor.evescn.com"]endpoint = ["https://harbor.evescn.com"]
EOF# 2.install cni
# PSK=`dd if=/dev/urandom count=48 bs=1 status=none | xxd -p -c 48`
# echo $PSK
kubectl apply -f ./flannel.yaml# 3.install necessary tools
# cd /opt/
# curl -o calicoctl -O -L "https://gh.api.99988866.xyz/https://github.com/containernetworking/plugins/releases/download/v0.9.0/cni-plugins-linux-amd64-v0.9.0.tgz"
# tar -zxvf cni-plugins-linux-amd64-v0.9.0.tgzfor i in $(docker ps -a --format "table {{.Names}}" | grep flannel)
doecho $idocker cp /opt/bridge $i:/opt/cni/bin/docker cp /usr/bin/ping $i:/usr/bin/pingdocker exec -it $i bash -c "sed -i -e 's/jp.archive.ubuntu.com\|archive.ubuntu.com\|security.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list"docker exec -it $i bash -c "apt-get -y update >/dev/null && apt-get -y install net-tools tcpdump lrzsz bridge-utils >/dev/null 2>&1"
done
flannel.yaml
配置文件
# flannel.yaml
---
kind: Namespace
apiVersion: v1
metadata:name: kube-flannellabels:pod-security.kubernetes.io/enforce: privileged
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: flannel
rules:
- apiGroups:- ""resources:- podsverbs:- get
- apiGroups:- ""resources:- nodesverbs:- list- watch
- apiGroups:- ""resources:- nodes/statusverbs:- patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: flannel
roleRef:apiGroup: rbac.authorization.k8s.iokind: ClusterRolename: flannel
subjects:
- kind: ServiceAccountname: flannelnamespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:name: flannelnamespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:name: kube-flannel-cfgnamespace: kube-systemlabels:tier: nodeapp: flannel
data:cni-conf.json: |{"name": "cbr0","cniVersion": "0.3.1","plugins": [{"type": "flannel","delegate": {"hairpinMode": true,"isDefaultGateway": true}},{"type": "portmap","capabilities": {"portMappings": true}}]}net-conf.json: |{"Network": "10.244.0.0/16","Backend": {"Type": "ipsec","PSK": "bd3d298e0f57599da780f7a2596719a96ac781ac21f496443bdbd14383810167864748a6bab6b9f1c4b039a3a240ba95"}}
---
apiVersion: apps/v1
kind: DaemonSet
metadata:name: kube-flannel-dsnamespace: kube-systemlabels:tier: nodeapp: flannel
spec:selector:matchLabels:app: flanneltemplate:metadata:labels:tier: nodeapp: flannelspec:affinity:nodeAffinity:requiredDuringSchedulingIgnoredDuringExecution:nodeSelectorTerms:- matchExpressions:- key: kubernetes.io/osoperator: Invalues:- linuxhostNetwork: truepriorityClassName: system-node-criticaltolerations:- operator: Existseffect: NoScheduleserviceAccountName: flannelinitContainers:- name: install-cni-pluginimage: harbor.dayuan1997.com/devops/rancher/mirrored-flannelcni-flannel-cni-plugin:v1.1.0command:- cpargs:- -f- /flannel- /opt/cni/bin/flannelvolumeMounts:- name: cni-pluginmountPath: /opt/cni/bin- name: install-cniimage: harbor.dayuan1997.com/devops/rancher/mirrored-flannelcni-flannel:v0.19.2command:- cpargs:- -f- /etc/kube-flannel/cni-conf.json- /etc/cni/net.d/10-flannel.conflistvolumeMounts:- name: cnimountPath: /etc/cni/net.d- name: flannel-cfgmountPath: /etc/kube-flannel/containers:- name: kube-flannelimage: harbor.dayuan1997.com/devops/rancher/mirrored-flannelcni-flannel:v0.19.2command:- /opt/bin/flanneldargs:- --ip-masq- --kube-subnet-mgrresources:requests:cpu: "100m"memory: "50Mi"limits:cpu: "100m"memory: "50Mi"securityContext:privileged: falsecapabilities:add: ["NET_ADMIN", "NET_RAW"]env:- name: POD_NAMEvalueFrom:fieldRef:fieldPath: metadata.name- name: POD_NAMESPACEvalueFrom:fieldRef:fieldPath: metadata.namespace- name: EVENT_QUEUE_DEPTHvalue: "5000"volumeMounts:- name: runmountPath: /run/flannel- name: flannel-cfgmountPath: /etc/kube-flannel/- name: xtables-lockmountPath: /run/xtables.lock- name: tunmountPath: /dev/net/tunvolumes:- name: tunhostPath:path: /dev/net/tun- name: runhostPath:path: /run/flannel- name: cni-pluginhostPath:path: /opt/cni/bin- name: cnihostPath:path: /etc/cni/net.d- name: flannel-cfgconfigMap:name: kube-flannel-cfg- name: xtables-lockhostPath:path: /run/xtables.locktype: FileOrCreate
flannel.yaml
参数解释
-
Backend.Type
- 含义: 用于指定
flannel
工作模式。 host-gw
:flannel
工作在host gw
模式。
- 含义: 用于指定
-
Backend.Type
- 含义: 用于指定
flannel
工作模式。 ipsec
:flannel
工作在ipsec
模式。
- 含义: 用于指定
-
Backend.PSK
- 含义: 用于指定
ipsec
模式对等共享密钥
字符串 。 xxxxxxxxx
:ipsec
模式中,node
节点之间数据转发使用对等共享密钥
信息。
- 含义: 用于指定
- 安装
k8s
集群和flannel
服务
# ./install.shCreating cluster "flannel-ipsec" ...✓ Ensuring node image (kindest/node:v1.23.4) 🖼✓ Preparing nodes 📦 📦 📦 ✓ Writing configuration 📜 ✓ Starting control-plane 🕹️ ✓ Installing StorageClass 💾 ✓ Joining worker nodes 🚜
Set kubectl context to "kind-flannel-ipsec"
You can now use your cluster with:kubectl cluster-info --context kind-flannel-ipsecThanks for using kind! 😊
- 查看安装的服务
root@kind:~# kubectl get pods -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-64897985d-glk45 1/1 Running 0 4m55s
kube-system coredns-64897985d-xtvbk 1/1 Running 0 4m55s
kube-system etcd-flannel-ipsec-control-plane 1/1 Running 0 5m9s
kube-system kube-apiserver-flannel-ipsec-control-plane 1/1 Running 0 5m8s
kube-system kube-controller-manager-flannel-ipsec-control-plane 1/1 Running 0 5m12s
kube-system kube-flannel-ds-9wxv4 1/1 Running 0 4m38s
kube-system kube-flannel-ds-c9mfm 1/1 Running 0 4m38s
kube-system kube-flannel-ds-z2mlw 1/1 Running 0 4m38s
kube-system kube-proxy-6tjxp 1/1 Running 0 4m55s
kube-system kube-proxy-l96h9 1/1 Running 0 4m39s
kube-system kube-proxy-vt9f4 1/1 Running 0 4m39s
kube-system kube-scheduler-flannel-ipsec-control-plane 1/1 Running 0 5m8s
local-path-storage local-path-provisioner-5ddd94ff66-h9r8t 1/1 Running 0 4m55s
k8s
集群安装 Pod
测试网络
root@kind:~# cat cni.yamlapiVersion: apps/v1
kind: DaemonSet
#kind: Deployment
metadata:labels:app: cniname: cni
spec:#replicas: 1selector:matchLabels:app: cnitemplate:metadata:labels:app: cnispec:containers:- image: harbor.dayuan1997.com/devops/nettool:0.9name: nettoolboxsecurityContext:privileged: true---
apiVersion: v1
kind: Service
metadata:name: serversvc
spec:type: NodePortselector:app: cniports:- name: cniport: 80targetPort: 80nodePort: 32000
root@kind:~# kubectl apply -f cni.yaml
daemonset.apps/cni created
service/serversvc createdroot@kind:~# kubectl run net --image=harbor.dayuan1997.com/devops/nettool:0.9
pod/net created
- 查看安装服务信息
root@kind:~# kubectl get pods -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
cni-88hb9 1/1 Running 0 11s 10.244.1.5 flannel-ipsec-worker2 <none> <none>
cni-cnjjv 1/1 Running 0 11s 10.244.2.2 flannel-ipsec-worker <none> <none>
net 1/1 Running 0 7s 10.244.2.3 flannel-ipsec-worker <none> <none>root@kind:~# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 5m38s
serversvc NodePort 10.96.27.120 <none> 80:32000/TCP 15s
三、测试网络
同节点 Pod
网络通讯
可以查看此文档 Flannel UDP 模式 中,同节点网络通讯,数据包转发流程一致
Flannel 同节点通信通过
l2
网络通信,2
层交换机完成
不同节点 Pod
网络通讯
Pod
节点信息
## ip 信息
root@kind:~# kubectl exec -it net -- ip a l
3: eth0@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1423 qdisc noqueue state UP group default link/ether 9e:65:44:01:03:01 brd ff:ff:ff:ff:ff:ff link-netnsid 0inet 10.244.2.3/24 brd 10.244.2.255 scope global eth0valid_lft forever preferred_lft foreverinet6 fe80::9c65:44ff:fe01:301/64 scope link valid_lft forever preferred_lft forever## 路由信息
root@kind:~# kubectl exec -it net -- ip r s
default via 10.244.2.1 dev eth0
10.244.0.0/16 via 10.244.2.1 dev eth0
10.244.2.0/24 dev eth0 proto kernel scope link src 10.244.2.3
Pod
节点所在Node
节点信息
root@kind:~# docker exec -it flannel-ipsec-worker bash## ip 信息
root@flannel-ipsec-worker:/# ip a l
2: cni0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1423 qdisc noqueue state UP group default qlen 1000link/ether ba:bf:a0:56:6e:bd brd ff:ff:ff:ff:ff:ffinet 10.244.2.1/24 brd 10.244.2.255 scope global cni0valid_lft forever preferred_lft foreverinet6 fe80::b8bf:a0ff:fe56:6ebd/64 scope link valid_lft forever preferred_lft forever
3: vethfbd871aa@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1423 qdisc noqueue master cni0 state UP group default link/ether 4e:e5:69:91:8f:8c brd ff:ff:ff:ff:ff:ff link-netns cni-5fcabfe2-dfb8-f91d-676a-31bdd629b2cdinet6 fe80::4ce5:69ff:fe91:8f8c/64 scope link valid_lft forever preferred_lft forever
4: veth3408e015@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1423 qdisc noqueue master cni0 state UP group default link/ether 66:d0:d9:c8:cc:ad brd ff:ff:ff:ff:ff:ff link-netns cni-c909ffba-a547-2acc-5daa-ce20a181b5c4inet6 fe80::64d0:d9ff:fec8:ccad/64 scope link valid_lft forever preferred_lft forever
7: eth0@if8: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default link/ether 02:42:ac:12:00:03 brd ff:ff:ff:ff:ff:ff link-netnsid 0inet 172.18.0.3/16 brd 172.18.255.255 scope global eth0valid_lft forever preferred_lft foreverinet6 fc00:f853:ccd:e793::3/64 scope global nodad valid_lft forever preferred_lft foreverinet6 fe80::42:acff:fe12:3/64 scope link valid_lft forever preferred_lft forever## 路由信息
root@flannel-ipsec-worker:/# ip r s
default via 172.18.0.1 dev eth0
10.244.2.0/24 dev cni0 proto kernel scope link src 10.244.2.1
172.18.0.0/16 dev eth0 proto kernel scope link src 172.18.0.3
Pod
节点进行ping
包测试,访问cni-88hb9
Pod
节点
root@kind:~# kubectl exec -it net -- pping 10.244.1.5 -c 1
PING 10.244.1.5 (10.244.1.5): 56 data bytes
64 bytes from 10.244.1.5: seq=0 ttl=62 time=0.839 ms--- 10.244.1.5 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 0.839/0.839/0.839 ms
Pod
节点eth0
网卡抓包
net~$ tcpdump -pne -i eth0
08:27:53.131272 9e:65:44:01:03:01 > ba:bf:a0:56:6e:bd, ethertype IPv4 (0x0800), length 98: 10.244.2.3 > 10.244.1.5: ICMP echo request, id 59, seq 0, length 64
08:27:53.131427 ba:bf:a0:56:6e:bd > 9e:65:44:01:03:01, ethertype IPv4 (0x0800), length 98: 10.244.1.5 > 10.244.2.3: ICMP echo reply, id 59, seq 0, length 64
数据包源 mac
地址: 9e:65:44:01:03:01
为 eth0
网卡 mac
地址,而目的 mac
地址: ba:bf:a0:56:6e:bd
为 net
Pod
节点 cni0
网卡对应的网卡 mac
地址,cni0
网卡 ip
地址为网络网关地址 10.244.2.1
, flannel
为 2
层网络模式通过路由送往数据到网关地址
net~$ arp -n
Address HWtype HWaddress Flags Mask Iface
10.244.2.1 ether ba:bf:a0:56:6e:bd C eth0
而通过 veth pair
可以确定 Pod
节点 eth0
网卡对应的 veth pair
为 veth3408e015@if3
网卡
flannel-ipsec-worker
节点veth3408e015
网卡抓包
root@flannel-ipsec-worker:/# tcpdump -pne -i veth3408e015
08:29:10.803335 9e:65:44:01:03:01 > ba:bf:a0:56:6e:bd, ethertype IPv4 (0x0800), length 98: 10.244.2.3 > 10.244.1.5: ICMP echo request, id 79, seq 0, length 64
08:29:10.803452 ba:bf:a0:56:6e:bd > 9e:65:44:01:03:01, ethertype IPv4 (0x0800), length 98: 10.244.1.5 > 10.244.2.3: ICMP echo reply, id 79, seq 0, length 64
因为他们互为 veth pair
所以抓包信息相同
flannel-ipsec-worker
节点cni0
网卡抓包
root@flannel-ipsec-worker:/# tcpdump -pne -i cni0
08:29:25.347918 9e:65:44:01:03:01 > ba:bf:a0:56:6e:bd, ethertype IPv4 (0x0800), length 98: 10.244.2.3 > 10.244.1.5: ICMP echo request, id 88, seq 0, length 64
08:29:25.348063 ba:bf:a0:56:6e:bd > 9e:65:44:01:03:01, ethertype IPv4 (0x0800), length 98: 10.244.1.5 > 10.244.2.3: ICMP echo reply, id 88, seq 0, length 64
数据包源 mac 地址: 9e:65:44:01:03:01
为 net
Pod
节点 eth0
网卡 mac
地址,而目的 mac 地址: ba:bf:a0:56:6e:bd
为 cni0
网卡 mac 地址
查看
flannel-ipsec-worker
主机路由信息,没有发现到达10.244.1.0/24
主机路由,ipsec
模式中需要使用以下命令查看
root@flannel-ipsec-worker:/# ip x s
src 172.18.0.3 dst 172.18.0.2proto esp spi 0xcfba8c29 reqid 11 mode tunnelreplay-window 0 flag af-unspecaead rfc4106(gcm(aes)) 0x5c6d833362f6676e74520fbb6074495753293433 128anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000
src 172.18.0.2 dst 172.18.0.3proto esp spi 0xce02fb42 reqid 11 mode tunnelreplay-window 32 flag af-unspecaead rfc4106(gcm(aes)) 0xbdc7124eb99d43e630025f0e5b84944dcefda130 128anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000
src 172.18.0.3 dst 172.18.0.4proto esp spi 0xcfc2f79d reqid 11 mode tunnelreplay-window 0 flag af-unspecaead rfc4106(gcm(aes)) 0x022427c450d510a2266ba5d732e55b163b92e7c8 128anti-replay context: seq 0x0, oseq 0x4, bitmap 0x00000000
src 172.18.0.4 dst 172.18.0.3proto esp spi 0xc91a08f3 reqid 11 mode tunnelreplay-window 32 flag af-unspecaead rfc4106(gcm(aes)) 0xbe4d33e6b634c9f9661eddff9c056bf97650d519 128anti-replay context: seq 0x4, oseq 0x0, bitmap 0x0000000f
src 172.18.0.3 dst 172.18.0.4proto esp spi 0xc767c36e reqid 11 mode tunnelreplay-window 0 flag af-unspecaead rfc4106(gcm(aes)) 0xef987aca2f456738cc3e6f4d27632d7bcfdb8ec1 128anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000
src 172.18.0.4 dst 172.18.0.3proto esp spi 0xc60be52f reqid 11 mode tunnelreplay-window 32 flag af-unspecaead rfc4106(gcm(aes)) 0x58d993430dc92b4969eb956c807413f375a88ec9 128anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000
src 172.18.0.3 dst 172.18.0.2proto esp spi 0xc42c113e reqid 11 mode tunnelreplay-window 0 flag af-unspecaead rfc4106(gcm(aes)) 0x1b4a451e10fe5f462fb54d8b0516c6eb30068f08 128anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000
src 172.18.0.2 dst 172.18.0.3proto esp spi 0xc01b7d3d reqid 11 mode tunnelreplay-window 32 flag af-unspecaead rfc4106(gcm(aes)) 0x4ed4c68a03a6445e219a889c7d9feb1ff358017a 128anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000root@flannel-ipsec-worker:/# ip x p
src 10.244.1.0/24 dst 10.244.2.0/24 dir fwd priority 0 tmpl src 172.18.0.4 dst 172.18.0.3proto esp reqid 11 mode tunnel
src 10.244.1.0/24 dst 10.244.2.0/24 dir in priority 0 tmpl src 172.18.0.4 dst 172.18.0.3proto esp reqid 11 mode tunnel
src 10.244.2.0/24 dst 10.244.1.0/24 dir out priority 0 tmpl src 172.18.0.3 dst 172.18.0.4proto esp reqid 11 mode tunnel
src 10.244.0.0/24 dst 10.244.2.0/24 dir fwd priority 0 tmpl src 172.18.0.2 dst 172.18.0.3proto esp reqid 11 mode tunnel
src 10.244.0.0/24 dst 10.244.2.0/24 dir in priority 0 tmpl src 172.18.0.2 dst 172.18.0.3proto esp reqid 11 mode tunnel
src 10.244.2.0/24 dst 10.244.0.0/24 dir out priority 0 tmpl src 172.18.0.3 dst 172.18.0.2proto esp reqid 11 mode tunnel
flannel-ipsec-worker
节点eth0
网卡抓包
搜索 esp
数据包, ipsec
模式下,数据包是密文 esp
数据包,需要进行解密后才能查看。
root@flannel-ipsec-worker:/# ip x s
src 172.18.0.3 dst 172.18.0.4proto esp spi 0xcfc2f79d reqid 11 mode tunnelreplay-window 0 flag af-unspecaead rfc4106(gcm(aes)) 0x022427c450d510a2266ba5d732e55b163b92e7c8 128anti-replay context: seq 0x0, oseq 0x4, bitmap 0x00000000
src 172.18.0.4 dst 172.18.0.3proto esp spi 0xc91a08f3 reqid 11 mode tunnelreplay-window 32 flag af-unspecaead rfc4106(gcm(aes)) 0xbe4d33e6b634c9f9661eddff9c056bf97650d519 128anti-replay context: seq 0x4, oseq 0x0, bitmap 0x0000000f
解密 esp
数据包,获取到如下图数据信息
- 内层数据包信息中,只有
ip
层信息和icmp
信息,没有mac
层信息,比较类似ipip
模式,ipip
数据包内层包也没有mac
信息,但是ipip
数据包非加密,不需要使用解密即可查看到包信息 - 外层数据包
ip
层地址使用node
节点地址,mac
地址为对应的ip
网卡mac
信息
root@flannel-ipsec-worker:/# arp -n
Address HWtype HWaddress Flags Mask Iface
172.18.0.4 ether 02:42:ac:12:00:04 C eth0
数据包流向
- 数据从
pod
服务发出,通过查看本机路由表,送往10.244.2.1
网卡。路由:10.244.0.0/16 via 10.244.2.1 dev eth0
- 通过
veth pair
网卡veth3408e015
发送数据到flannel-ipsec-worker
主机上,在转送到cni0: 10.244.2.1
网卡 flannel-ipsec-worker
主机查看自身ipsec
隧道信息后,会封装为ipsec
数据包并送往eth0
接口- 对端
flannel-ipsec-worker
主机接受到数据包后,发现这个是一个ipsec
数据包,将数据包内核模块处理。 - 解封装后发现目的地址为
10.244.1.5
,通过查看本机路由表,送往cni0
网卡。路由:10.244.1.0/24 dev cni0 proto kernel scope link src 10.244.1.1
- 通过
cni0
网卡brctl showmacs cni0
mac
信息 ,最终会把数据包送到cni-88hb9
主机
Service
网络通讯
可以查看此文档 Flannel UDP 模式 中,Service
网络通讯,数据包转发流程一致