1.部署Cilium网络组件
1.1 在k8s-master节点上，下载安装helm
wget https://mirrors.huaweicloud.com/helm/v3.15.2/helm-v3.15.2-linux-amd64.tar.gztar -zxvf helm-v3.15.2-linux-amd64.tar.gz
cp linux-amd64/helm /usr/bin/# helm version
version.BuildInfo{Version:"v3.15.2", GitCommit:"1a500d5625419a524fdae4b33de351cc4f58ec35", GitTreeState:"clean", GoVersion:"go1.22.4"}1.2 在任意k8s-master节点上，添加cilium安装源并下载安装包
# 添加安装源
helm repo add cilium https://helm.cilium.io# 下载安装包,运行pull命令后会获得最新版cilium的tar包
helm pull cilium/cilium
tar -xvf cilium-*.tar# 修改image为国内源
sed -i "s#quay.io/#m.daocloud.io/quay.io/#g" cilium/values.yaml1.3 在任意k8s-master节点上安装cilium
# helm install cilium ./cilium/ \
--namespace kube-system \
--set hubble.relay.enabled=true \
--set hubble.ui.enabled=true \
--set prometheus.enabled=true \
--set operator.prometheus.enabled=true \
--set hubble.enabled=true \
--set hubble.metrics.enabled="{dns,drop,tcp,flow,port-distribution,icmp,http}"
注：如需开启IPv6可添加--set ipv6.enabled=true参数# kubectl get pod -A -o wide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
kube-system cilium-87hd8 1/1 Running 0 4m53s 192.168.83.221 k8s-node02 <none> <none>
kube-system cilium-9fdbh 1/1 Running 0 4m53s 192.168.83.220 k8s-node01 <none> <none>
kube-system cilium-operator-f45f4975f-f9q7p 1/1 Running 0 4m53s 192.168.83.220 k8s-node01 <none> <none>
kube-system cilium-operator-f45f4975f-gw5z6 1/1 Running 0 4m53s 192.168.83.221 k8s-node02 <none> <none>
kube-system hubble-relay-84849f9dd5-59zhs 1/1 Running 0 4m53s 172.31.0.28 k8s-node01 <none> <none>
kube-system hubble-ui-79b7f9f4b-ccdrh 2/2 Running 0 4m53s 172.31.0.209 k8s-node01 <none> <none># kubectl get all --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system pod/cilium-87hd8 1/1 Running 0 4m20s
kube-system pod/cilium-9fdbh 1/1 Running 0 4m20s
kube-system pod/cilium-operator-f45f4975f-f9q7p 1/1 Running 0 4m20s
kube-system pod/cilium-operator-f45f4975f-gw5z6 1/1 Running 0 4m20s
kube-system pod/hubble-relay-84849f9dd5-59zhs 1/1 Running 0 4m20s
kube-system pod/hubble-ui-79b7f9f4b-ccdrh 2/2 Running 0 4m20sNAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
default service/kubernetes ClusterIP 10.66.0.1 <none> 443/TCP 136d
kube-system service/cilium-agent ClusterIP None <none> 9964/TCP 4m20s
kube-system service/hubble-metrics ClusterIP None <none> 9965/TCP 4m20s
kube-system service/hubble-peer ClusterIP 10.66.180.91 <none> 443/TCP 4m20s
kube-system service/hubble-relay ClusterIP 10.66.79.186 <none> 80/TCP 4m20s
kube-system service/hubble-ui ClusterIP 10.66.91.101 <none> 80/TCP 4m20sNAMESPACE NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
kube-system daemonset.apps/cilium 2 2 2 2 2 kubernetes.io/os=linux 4m20sNAMESPACE NAME READY UP-TO-DATE AVAILABLE AGE
kube-system deployment.apps/cilium-operator 2/2 2 2 4m20s
kube-system deployment.apps/hubble-relay 1/1 1 1 4m20s
kube-system deployment.apps/hubble-ui 1/1 1 1 4m20sNAMESPACE NAME DESIRED CURRENT READY AGE
kube-system replicaset.apps/cilium-operator-f45f4975f 2 2 2 4m20s
kube-system replicaset.apps/hubble-relay-84849f9dd5 1 1 1 4m20s
kube-system replicaset.apps/hubble-ui-79b7f9f4b 1 1 1 4m20s# kubectl get apiservices.apiregistration.k8s.io
NAME SERVICE AVAILABLE AGE
v1. Local True 136d
v1.admissionregistration.k8s.io Local True 136d
v1.apiextensions.k8s.io Local True 136d
v1.apps Local True 136d
v1.authentication.k8s.io Local True 136d
v1.authorization.k8s.io Local True 136d
v1.autoscaling Local True 136d
v1.batch Local True 136d
v1.certificates.k8s.io Local True 136d
v1.coordination.k8s.io Local True 136d
v1.discovery.k8s.io Local True 136d
v1.events.k8s.io Local True 136d
v1.flowcontrol.apiserver.k8s.io Local True 136d
v1.networking.k8s.io Local True 136d
v1.node.k8s.io Local True 136d
v1.policy Local True 136d
v1.rbac.authorization.k8s.io Local True 136d
v1.scheduling.k8s.io Local True 136d
v1.storage.k8s.io Local True 136d
v1alpha1.admissionregistration.k8s.io Local True 136d
v1alpha1.authentication.k8s.io Local True 136d
v1alpha1.internal.apiserver.k8s.io Local True 136d
v1alpha1.networking.k8s.io Local True 136d
v1alpha1.storage.k8s.io Local True 136d
v1alpha2.resource.k8s.io Local True 136d
v1beta1.admissionregistration.k8s.io Local True 136d
v1beta1.authentication.k8s.io Local True 136d
v1beta3.flowcontrol.apiserver.k8s.io Local True 136d
v2.autoscaling Local True 136d
v2.cilium.io Local True 90m
v2alpha1.cilium.io Local True 90m1.4 在任意k8s-master节点上安装cilium专属监控面板
下载部署文件
# wget https://github.com/cilium/cilium/blob/main/examples/kubernetes/addons/prometheus/monitoring-example.yaml替换镜像源
sed -i "s#docker.io/#dockerpull.com/#g" monitoring-example.yaml
sed -i "s/prom\/prometheus:v2.42.0/dockerpull.com\/prom\/prometheus:v2.42.0/g" monitoring-example.yaml部署监控
# kubectl apply -f monitoring-example.yaml
namespace/cilium-monitoring created
serviceaccount/prometheus-k8s created
configmap/grafana-config created
configmap/grafana-cilium-dashboard created
configmap/grafana-cilium-operator-dashboard created
configmap/grafana-hubble-dashboard created
configmap/grafana-hubble-l7-http-metrics-by-workload created
configmap/prometheus created
clusterrole.rbac.authorization.k8s.io/prometheus created
clusterrolebinding.rbac.authorization.k8s.io/prometheus created
service/grafana created
service/prometheus created
deployment.apps/grafana created
deployment.apps/prometheus created# kubectl get pod -A -o wide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
cilium-monitoring grafana-74b486577f-7w978 1/1 Running 0 37s 172.31.1.205 k8s-node02 <none> <none>
cilium-monitoring prometheus-58668c58df-jt9sn 1/1 Running 0 37s 172.31.1.40 k8s-node02 <none> <none>
kube-system cilium-87hd8 1/1 Running 0 60m 192.168.83.221 k8s-node02 <none> <none>
kube-system cilium-9fdbh 1/1 Running 0 60m 192.168.83.220 k8s-node01 <none> <none>
kube-system cilium-operator-f45f4975f-f9q7p 1/1 Running 0 60m 192.168.83.220 k8s-node01 <none> <none>
kube-system cilium-operator-f45f4975f-gw5z6 1/1 Running 0 60m 192.168.83.221 k8s-node02 <none> <none>
kube-system coredns-78d4595769-gl8nx 1/1 Running 0 3h37m 172.31.1.225 k8s-node02 <none> <none>
kube-system hubble-relay-84849f9dd5-59zhs 1/1 Running 0 60m 172.31.0.28 k8s-node01 <none> <none>
kube-system hubble-ui-79b7f9f4b-ccdrh 2/2 Running 0 60m 172.31.0.209 k8s-node01 <none> <none>1.5 在任意k8s-master节点，上将hubble-ui、grafan和prometheus的type修改为NodePort
# kubectl edit svc -n kube-system hubble-ui service/hubble-ui
# kubectl edit svc -n cilium-monitoring grafana service/grafana
# kubectl edit svc -n cilium-monitoring prometheus service/prometheus
将type: ClusterIP修改为type: NodePort# kubectl get svc -A | grep monitor
cilium-monitoring grafana NodePort 10.66.236.94 <none> 3000:32301/TCP 4h1m
cilium-monitoring prometheus NodePort 10.66.12.82 <none> 9090:30584/TCP 4h1m# kubectl get svc -A | grep hubble
kube-system hubble-metrics ClusterIP None <none> 9965/TCP 5h1m
kube-system hubble-peer ClusterIP 10.66.180.91 <none> 443/TCP 5h1m
kube-system hubble-relay ClusterIP 10.66.79.186 <none> 80/TCP 5h1m
kube-system hubble-ui NodePort 10.66.91.101 <none> 80:32093/TCP 5h1m2. 部署CoreDNS
2.1 在k8s-master节点上，创建CoreDNS配置文件
cat > /etc/kubernetes/yaml/coredns.yaml << EOF
# __MACHINE_GENERATED_WARNING__apiVersion: v1
kind: ServiceAccount
metadata:name: corednsnamespace: kube-systemlabels:kubernetes.io/cluster-service: "true"addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:labels:kubernetes.io/bootstrapping: rbac-defaultsaddonmanager.kubernetes.io/mode: Reconcilename: system:coredns
rules:
- apiGroups:- ""resources:- endpoints- services- pods- namespacesverbs:- list- watch
- apiGroups:- discovery.k8s.ioresources:- endpointslicesverbs:- list- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:annotations:rbac.authorization.kubernetes.io/autoupdate: "true"labels:kubernetes.io/bootstrapping: rbac-defaultsaddonmanager.kubernetes.io/mode: EnsureExistsname: system:coredns
roleRef:apiGroup: rbac.authorization.k8s.iokind: ClusterRolename: system:coredns
subjects:
- kind: ServiceAccountname: corednsnamespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:name: corednsnamespace: kube-systemlabels:addonmanager.kubernetes.io/mode: EnsureExists
data:Corefile: |.:53 {errorshealth {lameduck 5s}readykubernetes cluster.local in-addr.arpa ip6.arpa {pods insecurefallthrough in-addr.arpa ip6.arpattl 30}prometheus :9153forward . /etc/resolv.conf {max_concurrent 1000}cache 30loopreloadloadbalance}
---
apiVersion: apps/v1
kind: Deployment
metadata:name: corednsnamespace: kube-systemlabels:k8s-app: kube-dnskubernetes.io/cluster-service: "true"addonmanager.kubernetes.io/mode: Reconcilekubernetes.io/name: "CoreDNS"
spec:# replicas: not specified here:# 1. In order to make Addon Manager do not reconcile this replicas parameter.# 2. Default is 1.# 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on.strategy:type: RollingUpdaterollingUpdate:maxUnavailable: 1selector:matchLabels:k8s-app: kube-dnstemplate:metadata:labels:k8s-app: kube-dnsspec:securityContext:seccompProfile:type: RuntimeDefaultpriorityClassName: system-cluster-criticalserviceAccountName: corednsaffinity:podAntiAffinity:preferredDuringSchedulingIgnoredDuringExecution:- weight: 100podAffinityTerm:labelSelector:matchExpressions:- key: k8s-appoperator: Invalues: ["kube-dns"]topologyKey: kubernetes.io/hostnametolerations:- key: "CriticalAddonsOnly"operator: "Exists"nodeSelector:kubernetes.io/os: linuxcontainers:- name: corednsimage: registry.aliyuncs.com/google_containers/coredns/coredns:v1.11.1imagePullPolicy: IfNotPresentresources:limits:memory: 300Mirequests:cpu: 100mmemory: 70Miargs: [ "-conf", "/etc/coredns/Corefile" ]volumeMounts:- name: config-volumemountPath: /etc/corednsreadOnly: trueports:- containerPort: 53name: dnsprotocol: UDP- containerPort: 53name: dns-tcpprotocol: TCP- containerPort: 9153name: metricsprotocol: TCPlivenessProbe:httpGet:path: /healthport: 8080scheme: HTTPinitialDelaySeconds: 60timeoutSeconds: 5successThreshold: 1failureThreshold: 5readinessProbe:httpGet:path: /readyport: 8181scheme: HTTPsecurityContext:allowPrivilegeEscalation: falsecapabilities:add:- NET_BIND_SERVICEdrop:- ALLreadOnlyRootFilesystem: truednsPolicy: Defaultvolumes:- name: config-volumeconfigMap:name: corednsitems:- key: Corefilepath: Corefile
---
apiVersion: v1
kind: Service
metadata:name: kube-dnsnamespace: kube-systemannotations:prometheus.io/port: "9153"prometheus.io/scrape: "true"labels:k8s-app: kube-dnskubernetes.io/cluster-service: "true"addonmanager.kubernetes.io/mode: Reconcilekubernetes.io/name: "CoreDNS"
spec:selector:k8s-app: kube-dnsclusterIP: 10.66.0.2ports:- name: dnsport: 53protocol: UDP- name: dns-tcpport: 53protocol: TCP- name: metricsport: 9153protocol: TCPEOF2.2 应用CoreDNS配置文件
# kubectl apply -f /etc/kubernetes/yaml/coredns.yaml
serviceaccount/coredns created
clusterrole.rbac.authorization.k8s.io/system:coredns created
clusterrolebinding.rbac.authorization.k8s.io/system:coredns created
configmap/coredns created
deployment.apps/coredns created
service/kube-dns created# kubectl get pod -n kube-system -o wide | grep coredns
coredns-78d4595769-gl8nx 1/1 Running 0 164m 172.31.1.225 k8s-node02 <none> <none># kubectl get all --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system pod/cilium-87hd8 1/1 Running 0 4m20s
kube-system pod/cilium-9fdbh 1/1 Running 0 4m20s
kube-system pod/cilium-operator-f45f4975f-f9q7p 1/1 Running 0 4m20s
kube-system pod/cilium-operator-f45f4975f-gw5z6 1/1 Running 0 4m20s
kube-system pod/coredns-78d4595769-gl8nx 1/1 Running 0 162m
kube-system pod/hubble-relay-84849f9dd5-59zhs 1/1 Running 0 4m20s
kube-system pod/hubble-ui-79b7f9f4b-ccdrh 2/2 Running 0 4m20sNAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
default service/kubernetes ClusterIP 10.66.0.1 <none> 443/TCP 136d
kube-system service/cilium-agent ClusterIP None <none> 9964/TCP 4m20s
kube-system service/hubble-metrics ClusterIP None <none> 9965/TCP 4m20s
kube-system service/hubble-peer ClusterIP 10.66.180.91 <none> 443/TCP 4m20s
kube-system service/hubble-relay ClusterIP 10.66.79.186 <none> 80/TCP 4m20s
kube-system service/hubble-ui ClusterIP 10.66.91.101 <none> 80/TCP 4m20s
kube-system service/kube-dns ClusterIP 10.66.0.2 <none> 53/UDP,53/TCP,9153/TCP 135dNAMESPACE NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
kube-system daemonset.apps/cilium 2 2 2 2 2 kubernetes.io/os=linux 4m20sNAMESPACE NAME READY UP-TO-DATE AVAILABLE AGE
kube-system deployment.apps/cilium-operator 2/2 2 2 4m20s
kube-system deployment.apps/coredns 1/1 1 1 135d
kube-system deployment.apps/hubble-relay 1/1 1 1 4m20s
kube-system deployment.apps/hubble-ui 1/1 1 1 4m20sNAMESPACE NAME DESIRED CURRENT READY AGE
kube-system replicaset.apps/cilium-operator-f45f4975f 2 2 2 4m20s
kube-system replicaset.apps/coredns-78d4595769 1 1 1 135d
kube-system replicaset.apps/hubble-relay-84849f9dd5 1 1 1 4m20s
kube-system replicaset.apps/hubble-ui-79b7f9f4b 1 1 1 4m20s2.3 验证DNS解析是否正常
# dig -t a www.sohu.com @10.66.0.2; <<>> DiG 9.16.23 <<>> -t a www.sohu.com @10.66.0.2
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 64003
;; flags: qr rd ra; QUERY: 1, ANSWER: 3, AUTHORITY: 0, ADDITIONAL: 1;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags:; udp: 1408
;; QUESTION SECTION:
;www.sohu.com. IN A;; ANSWER SECTION:
www.sohu.com. 20 IN CNAME www.sohu.com.dsa.dnsv1.com.
www.sohu.com.dsa.dnsv1.com. 20 IN CNAME best.sched.d0-dk.tdnsdp1.cn.
best.sched.d0-dk.tdnsdp1.cn. 20 IN A 123.125.46.250;; Query time: 5 msec
;; SERVER: 10.66.0.2#53(10.66.0.2)
;; WHEN: Wed Jul 03 16:14:47 CST 2024
;; MSG SIZE rcvd: 138
