Complete Guide to Deploying Kubernetes 1.23.5 Cluster on CentOS 7

Tue, 31 May 2022 00:00:00 +0000

Environment Preparation

Disable Firewall and Mail Service

# Check firewall status
firewall-cmd --state
# Temporarily disable firewall
systemctl stop firewalld.service
# Disable firewall auto-start
systemctl disable firewalld.service

systemctl stop postfix.service
systemctl disable postfix.service

Disable SELinux

# Check SELinux status
getenforce
# Temporarily disable SELinux
setenforce 0
# Permanently disable SELinux
sed -i 's/^ *SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config

Disable Swap

# Temporarily disable swap
swapoff -a
# Permanently disable swap
sed -i.bak '/swap/s/^/#/' /etc/fstab
# Verify
free -g

Tuning Kernel Parameters and Modules

After system installation, appropriately adjusting kernel parameters based on practical application scenarios can help establish a more efficient and stable system environment. This includes:

Optimizing resource allocation parameters (e.g., modifying the maximum number of open file descriptors to improve high-concurrency service capabilities)
Adjusting network parameters using sysctl (e.g., modifying ip_local_port_range, nf_conntrack_max, and socket buffer sizes)
Managing kernel modules: Loading or disabling unnecessary modules (e.g., disabling unused hardware drivers to reduce potential vulnerabilities while enabling specific acceleration modules for performance enhancement)

Typical operation examples:

# Modify network port range
sysctl -w net.ipv4.ip_local_port_range="1024 65000"

# Adjust connection tracking table size
echo 1048576 > /proc/sys/net/netfilter/nf_conntrack_max

# Dynamically load zRAM module
modprobe zram

Key principle: Make targeted adjustments according to actual service requirements, using monitoring data as guidance while maintaining parameter modification records.

Enable ipvs

If ipvs is not enabled, iptables will be used for packet forwarding (less efficient). Recommended to enable ipvs for better performance.

cat <<EOF> /etc/sysconfig/modules/ipvs.modules
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF

# Load modules
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack_ipv4

# Install ipset package
yum install ipset -y
 
# Install management tool ipvsadm
yum install ipvsadm -y

Synchronize server time

`1`	`yum install chrony -y`

# Master node configuration
vim /etc/chrony.conf
server ntp.aliyun.com iburst

allow 192.168.142.0/24

# Slave node
vim /etc/chrony.conf
server controller iburst

1
2

systemctl restart chronyd.service
systemctl enable chronyd.service

Modify Hostname and Hosts (Optional)

hostnamectl set-hostname master

hostnamectl set-hostname node1

hostnamectl set-hostname node2


cat >> /etc/hosts << EOF
192.168.231.3 master
192.168.231.4 node1
EOF
echo -e "\033[32m [Host Resolution] ==> OK \033[0m"

System Log Optimization

1
2

mkdir /var/log/journal
mkdir /etc/systemd/journald.conf.d

cat > /etc/systemd/journald.conf.d/99-prophet.conf << EOF
[journal]
# Persist logs to disk
Storage=persistent
# Compress historical logs
Compress=yes

# Set sync interval to disk as 5 minutes
SyncIntervalSec=5m
# Control rate limiting for log messages
RateLimitInterval=30s
RateLimitBurst=1000

# Maximum disk usage 10G
SystemMaxUse=10G

# Control runtime log size
RuntimeMaxUse=500M

# Max individual log file size 200M
SystemMaxFileSize=200M

# Log retention period (2 weeks)
MaxRetentionSec=2week

# Disable syslog forwarding
ForwardToSyslog=no

# Enable sealing mode when size limits reached
SystemMaxSealing=yes
RuntimeMaxSealing=yes

# Enable watchdog timer (restart service after 1h inactivity)
RuntimeWatchdogSec=1h

# Enable automatic log file optimization
SystemMaxFilesTidy=yes
RuntimeMaxFilesTidy=yes

# Max age for individual log files
RuntimeMaxFileSec=7days

# Auto-select best storage mode
StorageAuto=yes

EOF

1
2

systemctl restart systemd-journald
echo -e "\033[32m [Log Optimization] ==> OK \033[0m"

Docker Installation

set -e
sudo yum install -y yum-utils
sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
sudo yum install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

sudo systemctl start docker
sudo systemctl enable docker

vim /etc/docker/daemon.json

{
    "exec-opts": ["native.cgroupdriver=systemd"],
    "insecure-registries": [
        "192.168.142.99"
    ]
}

1
2

sudo systemctl restart docker
docker login 192.168.142.99

Kubernetes Installation

Installing kubelet, kubeadm, and kubectl

Adding Kubernetes Repository

cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
        http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

Install kubeadm, kubelet, and kubectl

# Check version, the latest version is 1.23.5-0
yum list kubeadm --showduplicates | sort -r

yum install -y kubelet-1.23.5-0 kubectl-1.23.5-0 kubeadm-1.23.5-0

kubeadm version
# kubeadm version: &version.Info{Major:"1", Minor:"23", GitVersion:"v1.23.5", GitCommit:"c285e781331a3785a7f436042c65c5641ce8a9e9", GitTreeState:"clean", BuildDate:"2022-03-16T15:57:37Z", GoVersion:"go1.17.8", Compiler:"gc", Platform:"linux/amd64"}

Modify kubelet configuration

1
2

# Modify configuration file /etc/sysconfig/kubelet (this file does not exist by default and needs to be created)
KUBELET_EXTRA_ARGS=--root-dir=/var/lib/kubelet

Start kubelet service and enable it to start on boot

1
2

systemctl start kubelet
systemctl enable kubelet

Initialize Kubernetes Cluster

Initialization via Configuration File

cat <<EOF> kubeadm.yaml 
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.168.4.27 # Internal IP of the apiserver node
bindPort: 6443
nodeRegistration:
criSocket: /run/containerd/containerd.sock  # Switch to containerd
imagePullPolicy: IfNotPresent
name: master
taints:
- effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
type: CoreDNS # DNS type (CoreDNS)
etcd:
local:
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers # Modified image repository for accessibility
kind: ClusterConfiguration
kubernetesVersion: 1.23.5 # Kubernetes version
networking:
dnsDomain: cluster.local
podSubnet: 10.244.0.0/16  
serviceSubnet: 10.96.0.0/12
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs  # kube-proxy mode
EOF

`1`	`kubeadm init --config kubeadm.yaml`

📌 Important

kubeadm-installed certificates have a default validity period of one year

kube-proxy operates in iptables mode by default. This can be modified via kubectl edit configmap kube-proxy -n kube-system

Imperative Initialization

kubeadm init \
    --control-plane-endpoint k8svip:8443 \
    --kubernetes-version=v1.23.5 \
    --service-cidr=172.96.0.0/12 \
    --pod-network-cidr=172.244.0.0/16 \
    --image-repository registry.aliyuncs.com/google_containers \
    --upload-certs

kubeadm join k8svip:8443 --token i8zsn5.dakiqfxexdxn7wdt \
    --discovery-token-ca-cert-hash sha256:61ed5a0941ecf47078dac91c4389bc8abb9c761149e869a40f9c3da859b39dba \
    --control-plane --certificate-key fc133d520c12052c9391e075c3aa6dda456599b70b1335aba2c3e0680e75af6e

kubeadm init --kubernetes-version v1.23.5 \
    --apiserver-advertise-address 172.16.0.185 \
    --image-repository registry.aliyuncs.com/google_containers \
    --service-cidr 172.96.0.0/12 \
    --pod-network-cidr 172.244.0.0/16 \
    --upload-certs \
    --v=5

Install Calico Network Plugin (Execute on master node)

1
2
3

# Download manifest
curl https://docs.projectcalico.org/manifests/calico.yaml -o calico.yaml
kubectl apply -f calico.yaml

Installing Auto-Completion Tools

yum install -y bash-completion
source /usr/share/bash-completion/bash_completion
source <(kubectl completion bash)
echo "source <(kubectl completion bash)" >> ~/.bashrc

Deployment Verification Cluster

cat << EOF > nginx-ds.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: deploy-game
  namespace: default
spec:
  replicas: 8
  selector:
    matchLabels:
      app: game
      release: stabel
  template:
    metadata:
      labels:
        app: game
        release: stabel
        env: test
    spec:
      imagePullSecrets:
        - name: kkregcred
      containers:
      - name: game
        image: registry.cn-beijing.aliyuncs.com/kaikai136/docker-2048:v1
        imagePullPolicy: IfNotPresent
        ports:
        - name: http
          containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: game-svc
  namespace: default
spec:
  type: NodePort
  selector:
    app: game
    release: stabel
  ports:
  - name: http
    port: 80
    targetPort: 80
    nodePort: 32000
    protocol: TCP
EOF

kubectl apply -f nginx-ds.yaml

kubectl create deployment nginx –image=nginx kubectl expose deployment nginx –port=80 –type=NodePort kubectl get pod,svc

Testing Calico Network

`1`	`kubectl run busybox --image docker.io/library/busybox:1.28 --image-pull-policy=IfNotPresent --restart=Never --rm -it busybox -- sh`

/ # ping www.baidu.com
PING www.baidu.com (180.101.50.242): 56 data bytes
64 bytes from 180.101.50.242: seq=0 ttl=51 time=7.880 ms
64 bytes from 180.101.50.242: seq=1 ttl=51 time=7.247 ms
^C
# The network accessibility indicates Calico network plugin is properly installed
--- www.baidu.com ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max = 7.247/7.563/7.880 ms
/ # nslookup kubernetes.default.svc.cluster.local
Server:    172.96.0.10
Address 1: 172.96.0.10 kube-dns.kube-system.svc.cluster.local

Name:      kubernetes.default.svc.cluster.local
Address 1: 172.96.0.1 kubernetes.default.svc.cluster.local

# 172.96.0.10 corresponds to the clusterIP of coreDNS, verifying coreDNS configuration.
# Internal Service name resolution is handled by coreDNS.

# Note:
# Must use busybox version 1.28 as specified - latest versions may fail to resolve DNS and IP addresses with nslookup

Installing Additional Tools

Auto Completion Tools

yum install -y bash-completion
source /usr/share/bash-completion/bash_completion
source <(kubectl completion bash)
echo "source <(kubectl completion bash)" >> ~/.bashrc

Centos7 Configuration on Kang's Blog