最新版Kubernetes高可用部署教程v1.31.0
系统:Almalinux9
架构:
192.168.100.10 control-plane-endpoint.k8s.local control-plane-endpoint # 负载均衡服务器,配置负载均衡后修改ip后续会讲解
192.168.100.10 masterA.k8s.local masterA
192.168.100.20 masterB.k8s.local masterB
192.168.100.30 masterC.k8s.local masterC
192.168.100.11 workA.k8s.local workA
192.168.100.22 workB.k8s.local workB
192.168.100.33 workC.k8s.local workC
集群拓扑图:
环境配置(全部节点都需要同步,节点多可以采用Ansible)
IP地址、主机名配置以及dns映射
我这边采用nmtui图形化进行配置,大家根据习惯命令配置即可
[root@masterA ~]# vi /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.100.10 control-plane-endpoint.k8s.local control-plane-endpoint
192.168.100.10 masterA.k8s.local masterA
192.168.100.20 masterB.k8s.local masterB
192.168.100.30 masterC.k8s.local masterC
192.168.100.11 workA.k8s.local workA
192.168.100.22 workB.k8s.local workB
192.168.100.33 workC.k8s.local workC
12345678910
关闭selinux、防火墙、swap分区
# 关闭selinux
vi /etc/selinux/config
...
SELINUX=disabled
...
[root@masterA ~]# setenforce 0
# 关闭防火墙
systemctl disable --now firewalld
# 关闭swpa分区
[root@masterA ~]# swapoff -a
[root@masterA ~]# vi /etc/fstab
...
#
/dev/mapper/almalinux_192-root / xfs defaults 0 0
UUID=82c99b52-c337-40ba-8518-e46c1246eaea /boot xfs defaults 0 0
# /dev/mapper/almalinux_192-swap none swap defaults 0 0 # 注释本行
123456789101112131415161718
清空iptables以及配置时间同步
# 清空以及关闭iptables
[root@masterA ~]# iptables -F
[root@masterA ~]# iptables -X
[root@masterA ~]# iptables -Z
[root@masterA ~]# /usr/sbin/iptables-save
[root@masterA ~]# systemctl stop iptables
[root@masterA ~]# systemctl disable iptables
# 配置时间同步
[root@masterA ~]# vi /etc/chrony.conf
...
# pool 2.almalinux.pool.ntp.org iburst
server ntp1.aliyun.com iburst //增加的阿里云NTP服务
server ntp2.aliyun.com iburst //增加的阿里云NTP服务
...
[root@masterA ~]# systemctl restart chronyd
[root@masterA ~]# chronyc sources
MS Name/IP address Stratum Poll Reach LastRx Last sample
===============================================================================
^? 120.25.115.20 2 7 21 4 +1882us[+1882us] +/- 12ms
^* 203.107.6.88 2 6 35 10 -422us[ -452us] +/- 28ms
123456789101112131415161718192021
配置Yum源(阿里云)
# 替换原仓库地址为阿里云
[root@masterA ~]# sed -e 's|^mirrorlist=|#mirrorlist=|g'
-e 's|^# baseurl=https://repo.almalinux.org|baseurl=https://mirrors.aliyun.com|g'
-i.bak
/etc/yum.repos.d/almalinux*.repo
# 生成缓存
[root@masterA ~]# dnf makecache
# 更新系统
[root@masterA ~]# dnf update -y
1234567891011
安装常用软件
# 常用软件
[root@masterA ~]# dnf install epel-release net-tools htop bash-completion wget vim -y
# 修改history条数和kubectl命令自动补全
[root@masterA ~]# cat .bashrc
...
HISTSIZE=5000
HISTFILESIZE=5000
source <(kubectl completion bash)
123456789
配置加载内核以及ip转发
[root@masterA ~]# cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF
[root@masterA ~]# modprobe overlay
[root@masterA ~]# modprobe br_netfilter
# 设置所需的 sysctl 参数,参数在重新启动后保持不变
[root@masterA ~]# cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF
# 应用 sysctl 参数而不重新启动
[root@masterA ~]# sysctl --system
# 通过运行以下指令确认 br_netfilter 和 overlay 模块被加载
[root@masterA ~]# lsmod | grep br_netfilter
[root@masterA ~]# lsmod | grep overlay
# 通过运行以下指令确认 net.bridge.bridge-nf-call-iptables、net.bridge.bridge-nf-call-ip6tables 和 net.ipv4.ip_forward 系统变量在你的 sysctl 配置中被设置为 1
[root@masterA ~]# sysctl net.bridge.bridge-nf-call-iptables net.bridge.bridge-nf-call-ip6tables net.ipv4.ip_forward
123456789101112131415161718192021222324
创建第一个master节点
安装containerd
# 安装必要的一些系统工具
[root@masterA ~]# dnf install -y yum-utils device-mapper-persistent-data lvm2
# 添加软件源信息
[root@masterA ~]# yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# 修改仓库地址为阿里云
[root@masterA ~]# sed -i 's+download.docker.com+mirrors.aliyun.com/docker-ce+' /etc/yum.repos.d/docker-ce.repo
# 创建缓存
[root@masterA ~]# dnf makecache
# 安装containerd
[root@masterA ~]# dnf install containerd -y
# 启动containerd
[root@masterA ~]# systemctl enable --now containerd.service
# 生成containerd默认配置文件
[root@masterA ~]# containerd config default > /etc/containerd/config.toml
# 修改配置文件
[root@masterA ~]# vim /etc/containerd/config.toml
[plugins."io.containerd.grpc.v1.cri"]
...
sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.8" # 修改containerd镜像拉取地址为阿里云
...
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
...
SystemdCgroup = true # 启动Cgroup
# 重启containerd
[root@masterA ~]# systemctl daemon-reload
[root@masterA ~]# systemctl restart containerd.service
1234567891011121314151617181920212223242526
安装kubelet、kubeadm、kubectl
# 添加kubernetes仓库源
[root@masterA ~]# cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://pkgs.k8s.io/core:/stable:/v1.31/rpm/
enabled=1
gpgcheck=1
gpgkey=https://pkgs.k8s.io/core:/stable:/v1.31/rpm/repodata/repomd.xml.key
exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni
EOF
# 安装三个工具
[root@masterA ~]# yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
# 启动kubelet
[root@masterA ~]# systemctl enable --now kubelet
1234567891011121314
配置负载均衡(单独讲解)
Keepalived+Nginx 实现双负载均衡器高可用
Keepalived+HAProxy 实现双负载均衡器高可用
初始化集群
# 初始化集群
[root@masterA ~]# kubeadm init --apiserver-advertise-address=192.168.100.10 --control-plane-endpoint=control-plane-endpoint.k8s.local --image-repository=registry.aliyuncs.com/google_containers --pod-network-cidr=172.16.0.0/16 --service-cidr=10.92.0.0/16 --service-dns-domain=k8s.local --upload-certs --v=5
12
常用参数如下:
1、api地址
–apiserver-advertise-address=IP地址
API 服务器将公布其正在监听的 IP 地址.如果未设置,则将使用默认网络接口.
2、负载均衡器
–control-plane-endpoint=control-plane-endpoint.k8s.local
为控制平面指定一个稳定的 IP 地址或 DNS 名称.
3、镜像拉取地址
–image-repository=registry.aliyuncs.com/google_containers
选择容器注册表来从中提取控制平面映像.(默认“registry.k8s.io”)
4、work节点的IP范围
–pod-network-cidr=172.16.0.0/16
指定 Pod 网络的 IP 地址范围。如果设置,控制平面将自动为每个节点分配 CIDR。
5、service的IP范围
–service-cidr=10.92.0.0/16
对服务 VIP 使用备用 IP 地址范围.(默认“10.96.0.0/12”)
6、service的域名
–service-dns-domain=k8s.local
使用备用域名来提供服务,例如“myorg.internal”.(默认“cluster.local”)
7、上传证书
–upload-certs
将控制平面证书上传到 kubeadm-certs Secret.
验证集群
[root@masterA ~]# mkdir -p $HOME/.kube
[root@masterA ~]# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@masterA ~]# sudo chown $(id -u):$(id -g) $HOME/.kube/config
[root@masterA ~]# export KUBECONFIG=/etc/kubernetes/admin.conf
# 查看节点状态
[root@masterA ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
mastera.k8s.local NotReady control-plane 3h55m v1.31.0
# 目前状态都是NotReady那是因为网络插件还没安装后续安装完后就会正常
# 其次目前因为就只有一个节点所以就只有一个后续节点加入进来就会在这边展示
12345678910
添加第二、三master节点
步骤和创建第一个一致这个就不过多赘述,只需要做到安装kubelet、kubeadm、kubectl即可,如果是克隆的机子大家需要操作“kubeadm reset”重置集群后再操作加入集群哦~
# 加入集群,masterC也是如此
[root@masterB ~]# kubeadm join control-plane-endpoint.k8s.local:6443 --token bf37jf.cwkft399w4ggd2zh
--discovery-token-ca-cert-hash sha256:b8dc217fb0fff89493145846674fdb87841e496fd3b53329619924f632a17787
--control-plane --certificate-key 504bef590e2ba73544644236bac3bb82dbe61175f5be3708b6daad8ea53efa70
1234
添加work节点
work节点也跟master节点几乎一致在安装kubelet、kubeadm、kubectl的时候只用安装kubelet、kubeadm即可当然全部安装也不影响,一直做到安装kubelet、kubeadm、kubectl即可然后执行加入集群操作即可,如果是克隆的机子大家需要操作“kubeadm reset”重置集群后再操作加入集群哦~
# work节点加入集群,其他两个也是如此
[root@mworkA ~]# kubeadm join control-plane-endpoint.k8s.local:6443 --token bf37jf.cwkft399w4ggd2zh
--discovery-token-ca-cert-hash sha256:b8dc217fb0fff89493145846674fdb87841e496fd3b53329619924f632a17787
123
验证整个集群状态
# 查看node状态
[root@masterA ~]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
mastera.k8s.local NotReady control-plane 4h23m v1.31.0
masterb.k8s.local NotReady control-plane 3h58m v1.31.0
masterc.k8s.local NotReady control-plane 3h49m v1.31.0
worka.k8s.local NotReady <none> 177m v1.31.0
workb.k8s.local NotReady <none> 177m v1.31.0
workc.k8s.local NotReady <none> 177m v1.31.0
# 查看pod启动状态
[root@masterA ~]# kubectl get pods -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-6ddff5bd6d-57jz7 0/1 Pending 0 4h23m
kube-system coredns-6ddff5bd6d-pvzvx 0/1 Pending 0 4h23m
kube-system etcd-mastera.k8s.local 1/1 Running 21 (37m ago) 4h23m
kube-system etcd-masterb.k8s.local 1/1 Running 5 (32m ago) 3h58m
kube-system etcd-masterc.k8s.local 1/1 Running 2 (179m ago) 3h49m
kube-system kube-apiserver-mastera.k8s.local 1/1 Running 33 (35m ago) 4h23m
kube-system kube-apiserver-masterb.k8s.local 1/1 Running 5 (32m ago) 3h58m
kube-system kube-apiserver-masterc.k8s.local 1/1 Running 2 (179m ago) 3h49m
kube-system kube-controller-manager-mastera.k8s.local 1/1 Running 5 (130m ago) 4h23m
kube-system kube-controller-manager-masterb.k8s.local 1/1 Running 5 (32m ago) 3h58m
kube-system kube-controller-manager-masterc.k8s.local 1/1 Running 2 (179m ago) 3h49m
kube-system kube-proxy-4t8fp 1/1 Running 0 177m
kube-system kube-proxy-87tc4 1/1 Running 2 (179m ago) 3h49m
kube-system kube-proxy-mmcmn 1/1 Running 0 177m
kube-system kube-proxy-s92rn 1/1 Running 5 (32m ago) 3h58m
kube-system kube-proxy-vks8k 1/1 Running 0 177m
kube-system kube-proxy-xlqzh 1/1 Running 5 (130m ago) 4h23m
kube-system kube-scheduler-mastera.k8s.local 1/1 Running 5 (130m ago) 4h23m
kube-system kube-scheduler-masterb.k8s.local 1/1 Running 5 (32m ago) 3h58m
kube-system kube-scheduler-masterc.k8s.local 1/1 Running 2 (179m ago) 3h49m
1234567891011121314151617181920212223242526272829303132