K8s部署

K8s部署

部署前配置

1.关闭防火墙和selinux

2.做ssh免密

3.配置时钟同步

配置/etc/hosts并发送到全部节点

[root@master ~]# vim /etc/hosts
[root@master ~]# scp /etc/hosts root@node1:/etc/hosts
[root@master ~]# scp /etc/hosts root@node2:/etc/hosts

给所有节点配置镜像源和epel源

[root@master ~]# sed -e 's|^mirrorlist=|#mirrorlist=|g' \
-e 's|^#baseurl=http://dl.rockylinux.org/$contentdir|baseurl=https://mirrors.aliyun.com/rockylinux|g' \
-i.bak \
/etc/yum.repos.d/rocky-*.repo

[root@master ~]# vim /etc/yum.repos.d/epel.repo

[epel]
name=Extra Packages for Enterprise Linux $releasever - $basearch
# It is much more secure to use the #metalink, but if you wish to use a local mirror
# place its address here.
baseurl=https://repo.huaweicloud.com/epel/$releasever/Everything/$basearch/
#metalink=https://mirrors.fedoraproject.org/#metalink?repo=epel-$releasever&arch=$basearch&infra=$infra&content=$contentdir
enabled=1
gpgcheck=1
countme=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-$releasever
 
[epel-debuginfo]
name=Extra Packages for Enterprise Linux $releasever - $basearch - Debug
# It is much more secure to use the #metalink, but if you wish to use a local mirror
# place its address here.
baseurl=https://repo.huaweicloud.com/epel/$releasever/Everything/$basearch/debug/
#metalink=https://mirrors.fedoraproject.org/#metalink?repo=epel-debug-$releasever&arch=$basearch&infra=$infra&content=$contentdir
enabled=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-$releasever
gpgcheck=1
 
[epel-source]
name=Extra Packages for Enterprise Linux $releasever - $basearch - Source
# It is much more secure to use the #metalink, but if you wish to use a local mirror
# place its address here.
baseurl=https://repo.huaweicloud.com/epel/$releasever/Everything/source/tree/
#metalink=https://mirrors.fedoraproject.org/#metalink?repo=epel-source-$releasever&arch=$basearch&infra=$infra&content=$contentdir
enabled=0

修改系统最大打开文件数

[root@master ~]# vim /etc/security/limits.conf
#在最后加上
* soft nofile 65535
* hard nofile 65535
[root@master ~]# scp /etc/security/limits.conf root@node1:/etc/security/
[root@master ~]# scp /etc/security/limits.conf root@node2:/etc/security/

所有节点修改内核参数

[root@master ~]# vim /etc/sysctl.conf

net.ipv4.tcp_syncookies = 1 
net.ipv4.tcp_max_tw_buckets = 20480 
net.ipv4.tcp_max_syn_backlog = 20480
net.core.netdev_max_backlog = 262144
net.ipv4.tcp_fin_timeout = 20
[root@master ~]# sysctl -p

所有节点关闭swap

[root@master ~]# vim /etc/fstab
#UUID=bf3d506e-d88a-4354-92ca-4550f8516503 none                    swap    defaults        0 0
#注释这一条

[root@master01 ~]# swapoff -a

所有节点安装所需工具

[root@master ~]# yum install -y gcc autoconf sysstat

所有节点开启bridge网桥过滤

[root@master ~]# vim /etc/sysctl.d/k8s.conf

net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1

#加载br_netfilter模块并查看
[root@master ~]# modprobe br_netfilter && lsmod | grep br_netfilter

#加载配置文件
[root@master ~]# sysctl -p /etc/sysctl.d/k8s.conf

配置docker环境（所有）

[root@master ~]# yum install -y yum-utils
[root@master ~]# yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

安装指定版本

[root@master ~]# yum -y install docker-ce-25.0.5-1.el9

配置控制组

[root@master ~]# vim /etc/docker/daemon.json

{
        "registry-mirrors": [
                "https://docker.m.daocloud.io", 
                "https://dockerproxy.com",
                "https://docker.mirrors.ustc.edu.cn",
                "https://docker.nju.edu.cn"
 ],"exec-opts": ["native.cgroupdriver=systemd"]
}

[root@master ~]# systemctl daemon-reload
[root@master ~]# systemctl restart docker
[root@master ~]# systemctl enable docker

配置cri-docker

[root@master ~]# wget -c https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.9/cri-dockerd-0.3.9.amd64.tgz
[root@master ~]#tar -xzvf cri-dockerd-0.3.9.amd64.tgz --strip-components=1 -C /usr/local/bin/

下载service，socket文件

[root@master ~]# wget -O /etc/systemd/system/cri-docker.service https://raw.githubusercontent.com/Mirantis/cri-dockerd/master/packaging/systemd/cri-docker.service

[root@master ~]# wget -O /etc/systemd/system/cri-docker.socket https://raw.githubusercontent.com/Mirantis/cri-dockerd/master/packaging/systemd/cri-docker.socket

编辑/etc/systemd/system/cri-docker.service

[root@master ~]# /etc/systemd/system/cri-docker.service
#修改
ExecStart=/usr/local/bin/cri-dockerd --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.9 --container-runtime-endpoint=unix:///var/run/cri-dockerd.sock --cri-dockerd-root-directory=/var/lib/dockershim --cri-dockerd-root-directory=/var/lib/docker

编辑/etc/systemd/system/cri-docker.socket

[root@master ~]# vim /etc/systemd/system/cri-docker.socket
#修改
ListenStream=/var/run/cri-dockerd.sock

复制cri-dockerd-0.3.9.amd64.tgz到其他节点

[root@master01 ~]# scp cri-dockerd-0.3.9.amd64.tgz root@node1:/root/
[root@master01 ~]# scp cri-dockerd-0.3.9.amd64.tgz root@node2:/root/

node节点解压cri-docker

tar -xzvf cri-dockerd-0.3.9.amd64.tgz --strip-components=1 -C /usr/local/bin/

然后把master节点上的修改好的文件发送过去

[root@master ~]# scp /etc/systemd/system/cri-docker.s* root@node1:/etc/systemd/system/
[root@master ~]# scp /etc/systemd/system/cri-docker.s* root@node2:/etc/systemd/system/

[root@master ~]# systemctl daemon-reload
[root@master ~]# systemctl restart cri-docker
[root@master ~]# systemctl enable cri-docker

配置k8s源（所有）

[root@master ~]# vim /etc/yum.repos.d/k8s.repo

[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.28/rpm/
enabled=1
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.28/rpm/repodata/repomd.xml.key

安装所需软件包

[root@master ~]# yum install -y kubelet kubeadm kubectl

配置k8s cgroup控制组

[root@master~]# vim /etc/sysconfig/kubelet

KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"

[root@master01 ~]# systemctl enable kubelet

在master节点上打印所需的镜像文件

[root@master ~]# kubeadm config images list
[root@master ~]# kubeadm config print init-defaults > kubeadm-config.yaml

在master节点上修改参数

[root@master ~]# vim kubeadm-config.yaml

advertiseAddress: 192.168.100.100 #修改成master的ip

criSocket: unix:///var/run/cri-dockerd.sock #修改使用docker

name: master  #修改节点名称


imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers #修改使用阿里云镜像仓库

配置文件初始化

[root@master ~]# kubeadm init --config kubeadm-config.yaml --upload-certs

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.100.100:6443 --token abcdef.0123456789abcdef \
        --discovery-token-ca-cert-hash sha256:75ee27ec5f70b3b416bf39f86da3a85b94beb88a401a8b71514211595f5d9a79

在master节点上配置环境变量

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
  #如果是root用户则
export KUBECONFIG=/etc/kubernetes/admin.conf  

把node节点加入集群

kubeadm join 192.168.100.100:6443 --token abcdef.0123456789abcdef \
        --discovery-token-ca-cert-hash sha256:75ee27ec5f70b3b416bf39f86da3a85b94beb88a401a8b71514211595f5d9a79 --cri-socket=unix:///var/run/cri-dockerd.sock

在master节点上下载calico文件

[root@master ~]# wget https://raw.githubusercontent.com/projectcalico/calico/v3.24.1/manifests/calico.yaml

在master节点上创建calico

[root@master ~]# kubectl apply -f calico.yaml

查看集群节点状态

[root@master ~]# kubectl get nodes
NAME     STATUS   ROLES           AGE     VERSION
master   Ready    control-plane   3h26m   v1.28.15
node1    Ready    <none>          3h22m   v1.28.15
node2    Ready    <none>          3h21m   v1.28.15

查看k8s集群组件

[root@master ~]# kubectl get pod -n kube-system
NAME                                      READY   STATUS    RESTARTS      AGE
calico-kube-controllers-9d57d8f49-mndtc   1/1     Running   1 (34m ago)   3h20m
calico-node-n954f                         1/1     Running   1 (34m ago)   3h20m
calico-node-s4mtv                         1/1     Running   1 (34m ago)   3h20m
calico-node-sk2bk                         1/1     Running   1 (34m ago)   3h20m
coredns-6554b8b87f-mnw6s                  1/1     Running   1 (34m ago)   3h26m
coredns-6554b8b87f-zj9hd                  1/1     Running   1 (34m ago)   3h26m
etcd-master                               1/1     Running   1 (34m ago)   3h26m
kube-apiserver-master                     1/1     Running   1 (34m ago)   3h26m
kube-controller-manager-master            1/1     Running   1 (34m ago)   3h26m
kube-proxy-76llz                          1/1     Running   1 (34m ago)   3h22m
kube-proxy-mkf5f                          1/1     Running   1 (34m ago)   3h26m
kube-proxy-ztbz2                          1/1     Running   1 (34m ago)   3h22m
kube-scheduler-master                     1/1     Running   1 (34m ago)   3h26m