目录
第1章 安装前准备
1.1 节点规划
1.2 配置NTP
1.3 bind安装DNS服务
1.4 修改主机DNS
1.5 安装runtime环境及依赖
1.5.1 安装docker运行时
1.5.2 安装containerd运行时
1.6 安装habor仓库
1.7 配置高可用
第2章 k8s安装集群master
2.1 下载二进制安装文件
2.2 生成证书
2.2.1 生成etcd证书
2.2.2 生成k8s证书
2.3 安装etcd
2.4 安装apisercer
2.5 安装controller
2.6 安装scheduler
第3章 k8s安装集群node
3.1 TLS Bootstrapping配置
3.2 node证书配置
3.3 安装kubelet
3.4 安装kube-proxy
第4章 k8s安装集群插件
4.1 安装calico
4.2 安装coredns
4.3 安装metrics-server
4.4 安装ingress-nginx
4.5 配置storageClass动态存储
4.6 安装kubesphere集群管理
第1章 安装前准备
1.1 节点规划
操作系统 Almalinux RHEL-8
m1,node1 : 192.168.44.201 主机名称：k8s-
m2,node2 : 192.168.44.202 主机名称：k8s-
m3,node3 : 192.168.44.203 主机名称：k8s-
| 192.168.44.201 | apiserver，controller，scheduler，kubelet，kube-proxy，flannel |
| 192.168.44.202 | apiserver，controller，scheduler，kubelet，kube-proxy，flannel |
| 192.168.44.203 | apiserver，controller，scheduler，kubelet，kube-proxy，flannel |
| 192.168.44.200 | vip |
| 10.0.0.0/16 | service网络 |
| 10.244.0.0/16 | pod网络 |
#配置免密登录201 cd ~ cat >host_list.txt </etc/sysctl.d/k8s.conf < 1.2 配置NTP
#服务端201 yum install chrony -y vim /etc/chrony.conf allow 192.168.44.0/24 systemctl enable chronyd.service systemctl restart chronyd.service chronyc sources #客户端所有master和node节点202.203 #批量201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do ssh root@${i} yum install chrony -y; done #单机编辑201 cp /etc/chrony.conf /tmp vim /tmp/chrony.conf server 192.168.44.201 iburst #分发201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'|grep -v "192.168.44.201"); do rsync -avzP /tmp/chrony.conf root@${i}:/etc/; ssh root@${i} cat /etc/chrony.conf; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done #批量重启服务201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'|grep -v "192.168.44.201"); do ssh root@${i} systemctl enable chronyd.service; ssh root@${i} systemctl restart chronyd.service; ssh root@${i} chronyc sources; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done1.3 bind安装DNS服务
#201 ~]# yum install -y bind 主配置文件 注意语法（分号空格）这里的IP是内网IP ~]# vim /etc/named.conf //确保以下配置正确 listen-on port 53 { 192.168.44.201; }; # 监听端口53 下面一行本来有ipv6地址 需要删除 directory "/var/named"; allow-query { any; }; #允许内机器都可以查 forwarders { 192.168.44.2; }; # 虚拟机这里填网关地址，阿里云机器可以填223.5.5.5 recursion yes; # 采用递归方法查询IP dnssec-enable no; dnssec-validation no; ~]# named-checkconf # 检查配置 没有信息即为正确 在 201 配置域名文件 增加两个zone配置，od.com为业务域，host.com为主机域 ~]# vim /etc/named.rfc1912.zones zone "host.com" IN { type master; file "host.com.zone"; allow-update { 192.168.44.201; }; }; zone "od.com" IN { type master; file "od.com.zone"; allow-update { 192.168.44.201; }; }; 在 201 配置主机域文件 第四行时间需要修改 （年月日01）每次修改配置文件都需要前滚一个序列号 ~]# vim /var/named/host.com.zone $ORIGIN host.com. $TTL 600 ; 10 minutes @ IN SOA dns.host.com. dnsadmin.host.com. ( 2021071301 ; serial 10800 ; refresh (3 hours) 900 ; retry (15 minutes) 604800 ; expire (1 week) 86400 ; minimum (1 day) ) NS dns.host.com. $TTL 60 ; 1 minute dns A 192.168.44.201 k8s-192-168-44-201 A 192.168.44.201 k8s-192-168-44-202 A 192.168.44.202 k8s-192-168-44-203 A 192.168.44.203 在 201 配置业务域文件 ~]# vim /var/named/od.com.zone $ORIGIN od.com. $TTL 600 ; 10 minutes @ IN SOA dns.od.com. dnsadmin.od.com. ( 2021071301 ; serial 10800 ; refresh (3 hours) 900 ; retry (15 minutes) 604800 ; expire (1 week) 86400 ; minimum (1 day) ) NS dns.od.com. $TTL 60 ; 1 minute dns A 192.168.44.201 在 201 启动bind服务，并测试 ~]# named-checkconf # 检查配置文件 ~]# systemctl start named ; systemctl enable named ~]# dig -t A k8s-192-168-44-203.host.com @192.168.44.201 +short #检查是否可以解析 192.168.44.2031.4 修改主机DNS
#所有master和node节点201.202.203 #批量201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do ssh root@${i} sed -ri '"s#(DNS1=)(.*)#1192.168.44.201#g"' /etc/sysconfig/network-scripts/ifcfg-eth0; ssh root@${i} systemctl restart NetworkManager.service; ssh root@${i} ping www.baidu.com -c 1; ssh root@${i} ping k8s-192-168-44-203.host.com -c 1; ssh root@${i} cat /etc/resolv.conf; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done ============================================================== # Generated by NetworkManager search host.com #添加后解析主机A记录 可以不加域名 例如 dig -t A k8s-192-168-44-201 @192.168.44.201 +short nameserver 192.168.44.2011.5 安装runtime环境及依赖
1.5.1 安装docker运行时
虽然docker被k8s抛弃了。但是还是要安装上，因为后期的ci/cd功能需要用到宿主机的docker的引擎来构建镜像，所以所有master和node节点201.202.203都需要安装上
#所有master和node节点201.202.203 #依赖关系：https://git.k8s.io/kubernetes/build/dependencies.yaml #官方网站：https://docs.docker.com/engine/install/centos/ #清理环境 yum remove docker docker-client docker-client-latest docker-common docker-latest docker-latest-logrotate docker-logrotate docker-engine #官方源（推荐） #批量201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do ssh root@${i} curl https://download.docker.com/linux/centos/docker-ce.repo -o /etc/yum.repos.d/docker-ce.repo; ssh root@${i} ls -l /etc/yum.repos.d; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done ===================================================================== curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo #安装指定版本docker这里是20.10.14版本 #批量201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do ssh root@${i} yum install -y yum-utils; ssh root@${i} yum list docker-ce --showduplicates | sort -r; ssh root@${i} yum list docker-ce-cli --showduplicates | sort -r; ssh root@${i} yum list containerd.io --showduplicates | sort -r; ssh root@${i} yum install docker-ce-20.10.14 docker-ce-cli-20.10.14 containerd.io -y; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done #阿里源（备用） #批量201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do ssh root@${i} curl http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -o /etc/yum.repos.d/docker-ce.repo; ssh root@${i} ls -l /etc/yum.repos.d; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done ================================================================== curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo #创建文件夹所有master和node节点201.202.203 #批量201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do ssh root@${i} mkdir /etc/docker -p; ssh root@${i} mkdir /data/docker -p; ssh root@${i} tree /data; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done #修改docker配置文件，bip172.xx.xx.1的xx.xx替换为主机ip的后两位 #单机编辑201 vim /etc/docker/daemon.json { "graph": "/data/docker", "storage-driver": "overlay2", "insecure-registries": ["registry.access.redhat.com","quay.io","192.168.44.201:8088"], "registry-mirrors": ["https://uoggbpok.mirror.aliyuncs.com"], "bip": "172.16.0.1/16", "exec-opts": ["native.cgroupdriver=systemd"], "live-restore": true, "log-driver": "json-file", "log-opts": {"max-size":"100m", "max-file":"3"} } #分发201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do rsync -avzP /etc/docker/daemon.json root@${i}:/etc/docker/; ssh root@${i} cat /etc/docker/daemon.json; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done #所有master和node节点201.202.203 #启动docker，验证版本信息 #批量201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do ssh root@${i} systemctl daemon-reload; ssh root@${i} systemctl enable docker; ssh root@${i} systemctl start docker; ssh root@${i} docker version; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done1.5.2 安装containerd运行时
注意：默认kubelet使用的是docker运行时，如果要指定其他运行时需要修改kebelet启动配置文件参数
KUBELET_RUNTIME_ARGS="--container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock"
#所有master和node节点201.202.203 #官方文档：https://kubernetes.io/zh/docs/setup/production-environment/container-runtimes/#containerd #设置模块所有master和node节点201.202.203 #单机编辑201 cat <> ~/.bashrc #分发201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do rsync -avzP ~/.bashrc root@${i}:~/; ssh root@${i} cat ~/.bashrc; ssh root@${i} systemctl restart containerd.service; ssh root@${i} crictl --version; ssh root@${i} crictl images; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done ======================================================================= crictl version v1.22.0 #简单入门使用方法见文档： https://kubernetes.io/zh/docs/tasks/debug-application-cluster/crictl/ 1.6 安装habor仓库
#前提准备201 yum install -y docker-compose sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose sudo chmod +x /usr/local/bin/docker-compose mkdir /opt/tool cd /opt/tool #官方文档：https://goharbor.io/docs/2.0.0/install-config/configure-https/ wget https://github.com/goharbor/harbor/releases/download/v2.0.1/harbor-offline-installer-v2.0.1.tgz mkdir -p /opt/harbor_data tar xf harbor-offline-installer-v2.0.1.tgz cd harbor mkdir cert -p cd cert #生成 CA 证书私钥 openssl genrsa -out ca.key 4096 #生成 CA 证书（替换CN为registry.myharbor.com） openssl req -x509 -new -nodes -sha512 -days 36500 -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=registry.myharbor.com" -key ca.key -out ca.crt #生成域名私钥 openssl genrsa -out registry.myharbor.com.key 4096 #生成域名请求csr（替换CN为registry.myharbor.com） openssl req -sha512 -new -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=registry.myharbor.com" -key registry.myharbor.com.key -out registry.myharbor.com.csr #生成 v3 扩展文件 cat > v3.ext <<-EOF authorityKeyIdentifier=keyid,issuer basicConstraints=CA:FALSE keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment extendedKeyUsage = serverAuth subjectAltName = @alt_names [alt_names] DNS.1=registry.myharbor.com DNS.2=registry.myharbor DNS.3=hostname IP.1=192.168.44.201 EOF #生成域名证书 openssl x509 -req -sha512 -days 36500 -extfile v3.ext -CA ca.crt -CAkey ca.key -CAcreateserial -in registry.myharbor.com.csr -out registry.myharbor.com.crt #查看效果 tree ./ ./ ├── ca.crt ├── ca.key ├── ca.srl ├── registry.myharbor.com.crt ├── registry.myharbor.com.csr ├── registry.myharbor.com.key └── v3.ext 0 directories, 7 files #转换xxx.crt为xxx.cert, 供 Docker 使用 openssl x509 -inform PEM -in registry.myharbor.com.crt -out registry.myharbor.com.cert #查看 tree ./ ./ ├── ca.crt ├── ca.key ├── ca.srl ├── registry.myharbor.com.cert ├── registry.myharbor.com.crt ├── registry.myharbor.com.csr ├── registry.myharbor.com.key └── v3.ext 0 directories, 8 files #复制证书到harbor主机的docker文件夹 mkdir /etc/docker/certs.d/registry.myharbor.com/ -p cp registry.myharbor.com.cert /etc/docker/certs.d/registry.myharbor.com/ cp registry.myharbor.com.key /etc/docker/certs.d/registry.myharbor.com/ cp ca.crt /etc/docker/certs.d/registry.myharbor.com/ 如果您将默认nginx端口 443 映射到不同的端口，请创建文件夹/etc/docker/certs.d/yourdomain.com:port或/etc/docker/certs.d/harbor_IP:port. #检查 tree /etc/docker/certs.d/ /etc/docker/certs.d/ └── registry.myharbor.com ├── ca.crt ├── registry.myharbor.com.cert └── registry.myharbor.com.key 1 directory, 3 files #重启Docker 引擎。 systemctl restart docker #部署或重新配置 Harbor cd /opt/tool/harbor/ cp harbor.yml.tmpl harbor.yml 修改配置文件主要修改参数如下： vim harbor.yml hostname = registry.myharbor.com //修改主机名称 关于http配置（见下） http: # port for http, default is 80. If https enabled, this port will redirect to https port port: 8088 //修改端口 关于https配置（见下） https: # https port for harbor, default is 443 port: 443 # The path of cert and key files for nginx certificate: /opt/tool/harbor/cert/registry.myharbor.com.crt private_key: /opt/tool/harbor/cert/registry.myharbor.com.key 其余配置 harbor_admin_password: 888888 //修改admin密码 data_volume: /opt/harbor_data //修改持久化数据目录 第五步：执行./install.sh #如果前面已经安装了http只要启用https的话操作如下 ./prepare docker-compose down -v docker-compose up -d 第六步：浏览器访问https://192.168.44.201或者https://registry.myharbor.com/ 账号admin密码888888注意：需要使用该仓库域名的机器配置hosts解析，这里为所有k8s的node节点
#所有master和node节点201.202.203 #单机编辑201 vim /etc/hosts 192.168.44.201 registry.myharbor.com #分发201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do rsync -avzP /etc/hosts root@${i}:/etc/; ssh root@${i} cat /etc/hosts; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done1.7 配置高可用
#201 && 202 && 203 配置Nginx反向代理192.168.44.201:6443和192.168.44.202:6443和192.168.44.203:6443 cat >/etc/yum.repos.d/nginx.repo </server/scripts/check_web.sh < /etc/keepalived/keepalived.conf < /server/scripts/check_web.sh < /server/scripts/check_ip.sh < /dev/null if [ $? -eq 0 ] then echo "主机$(hostnamectl |awk 'NR==1''{print $3}')里边的keepalived服务出现异常,请进行检查"|mail -s 异常告警-keepalived 2744588786@qq.com fi EOF chmod +x /server/scripts/check_ip.sh 配置发送邮件 vim /etc/mail.rc set from=xxxx@163.com smtp=smtp.163.com set smtp-auth-user=xxxx@163.com smtp-auth-password=xxxx smtp-auth=login systemctl restart postfix.service echo "邮件发送测试"|mail -s "邮件测试" xxxx@qq.com cat >/etc/keepalived/keepalived.conf < /server/scripts/check_web.sh < /server/scripts/check_ip.sh < /dev/null if [ $? -eq 0 ] then echo "主机$(hostnamectl |awk 'NR==1''{print $3}')里边的keepalived服务出现异常,请进行检查"|mail -s 异常告警-keepalived 2744588786@qq.com fi EOF chmod +x /server/scripts/check_ip.sh 配置发送邮件 vim /etc/mail.rc set from=xxxx@163.com smtp=smtp.163.com set smtp-auth-user=xxxx@163.com smtp-auth-password=xxxx smtp-auth=login systemctl restart postfix.service echo "邮件发送测试"|mail -s "邮件测试" xxxx@qq.com cat >/etc/keepalived/keepalived.conf < >/etc/sysctl.conf sysctl -p vim /etc/nginx/nginx.conf stream { log_format proxy '$time_local|$remote_addr|$upstream_addr|$protocol|$status|' '$session_time|$upstream_connect_time|$bytes_sent|$bytes_received|' '$upstream_bytes_sent|$upstream_bytes_received' ; upstream kube-apiserver { server 192.168.44.201:6443 max_fails=3 fail_timeout=30s; server 192.168.44.202:6443 max_fails=3 fail_timeout=30s; server 192.168.44.203:6443 max_fails=3 fail_timeout=30s; } server { listen 192.168.44.200:7443; proxy_connect_timeout 1s; proxy_timeout 900s; proxy_pass kube-apiserver; access_log /var/log/nginx/proxy.log proxy; } } systemctl restart nginx.service netstat -tunpl|grep nginx 第2章 k8s安装集群master
2.1 下载二进制安装文件
k8s下载
https://github.com/kubernetes/kubernetes/tree/master/CHANGELOG
https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.23.md
#（所有master和 node节点）201.202.203 #批量201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do ssh root@${i} mkdir /opt/tool -p; ssh root@${i} cd /opt/tool; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done #（所有master和 node节点）201.202.203 #单机下载201 cd /opt/tool wget https://storage.googleapis.com/kubernetes-release/release/v1.23.5/kubernetes-server-linux-amd64.tar.gz -O kubernetes-v1.23.5-server-linux-amd64.tar.gz tar xf kubernetes-v1.23.5-server-linux-amd64.tar.gz mv kubernetes /opt/kubernetes-1.23.5 #分发201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do rsync -avzP /opt/kubernetes-1.23.5 root@${i}:/opt/; ssh root@${i} ln -s /opt/kubernetes-1.23.5 /opt/kubernetes; ssh root@${i} tree -L 1 /opt/; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done #（所有master和 node节点）201.202.203 tree -L 1 /opt/ /opt/ ├── containerd ├── kubernetes -> /opt/kubernetes-1.23.5 ├── kubernetes-1.23.5 └── tool 4 directories, 0 files #命令补全（所有master和 node节点）201.202.203 #单机执行201 cp /opt/kubernetes/server/bin/kubectl /usr/local/bin/ source <(kubectl completion bash) && echo 'source <(kubectl completion bash)' >> ~/.bashrc #分发201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do rsync -avzP /usr/local/bin/kubectl root@${i}:/usr/local/bin/; rsync -avzP ~/.bashrc root@${i}:~/; ssh root@${i} tree -L 1 /usr/local/bin/; ssh root@${i} cat ~/.bashrc; ssh root@${i} kubectl version; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done =========================================================================== kubectl version Client Version: version.Info{Major:"1", Minor:"23", GitVersion:"v1.23.5", GitCommit:"c285e781331a3785a7f436042c65c5641ce8a9e9", GitTreeState:"clean", BuildDate:"2022-03-16T15:58:47Z", GoVersion:"go1.17.8", Compiler:"gc", Platform:"linux/amd64"} The connection to the server localhost:8080 was refused - did you specify the right host or port?
下载etcd
https://github.com/etcd-io/etcd/releases/tag/v3.5.1
依赖关系：https://git.k8s.io/kubernetes/build/dependencies.yaml
#所有etcd节点201.202.203 #批量201 cd ~ for i in 192.168.44.{201..203}; do ssh root@${i} mkdir /opt/tool -p; ssh root@${i} cd /opt/tool ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done #下载安装包所有etcd节点201.202.203 #单机下载201 wget https://github.com/etcd-io/etcd/releases/download/v3.5.1/etcd-v3.5.1-linux-amd64.tar.gz tar xf etcd-v3.5.1-linux-amd64.tar.gz mv etcd-v3.5.1-linux-amd64 /opt/etcd-v3.5.1 #分发201 cd ~ for i in 192.168.44.{201..203}; do rsync -avzP /opt/etcd-v3.5.1 root@${i}:/opt/; ssh root@${i} ln -s /opt/etcd-v3.5.1 /opt/etcd; ssh root@${i} tree -L 1 /opt/; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done #所有etcd节点201.202.203 tree -L 1 /opt/ /opt/ ├── containerd ├── etcd -> /opt/etcd-v3.5.1 ├── etcd-v3.5.1 ├── kubernetes -> /opt/kubernetes-1.23.5 ├── kubernetes-1.23.5 └── tool 6 directories, 0 files #拷贝命令到环境变量（所有etcd节点201.202.203） #单机执行201 cp /opt/etcd/etcdctl /usr/local/bin/ #分发201 cd ~ for i in 192.168.44.{201..203}; do rsync -avzP /usr/local/bin/etcdctl root@${i}:/usr/local/bin/; ssh root@${i} tree -L 1 /usr/local/bin/; ssh root@${i} etcdctl version; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done ==================================================================== etcdctl version etcdctl version: 3.5.1 API version: 3.52.2 生成证书
#201 cd rz curl -L https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssl_1.5.0_linux_amd64 -o cfssl curl -L https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssljson_1.5.0_linux_amd64 -o cfssl-json curl -L https://github.com/cloudflare/cfssl/releases/download/v1.5.0/cfssl-certinfo_1.5.0_linux_amd64 -o cfssl-certinfo chmod +x cfssl* mv cfssl* /usr/local/bin/ ll /usr/local/bin/ total 140556 -rwxr-xr-x 1 root root 12049979 Apr 29 20:53 cfssl -rwxr-xr-x 1 root root 12021008 Apr 29 21:16 cfssl-certinfo -rwxr-xr-x 1 root root 9663504 Apr 29 21:08 cfssl-json -rwxr-xr-x 1 1000 users 45611187 Aug 5 2021 crictl -rwxr-xr-x 1 root root 17981440 Apr 29 19:11 etcdctl -rwxr-xr-x 1 root root 46596096 Apr 29 18:24 kubectl
2.2.1 生成etcd证书
#创建证书目录（所有etcd节点）201.202.203 #批量201 cd ~ for i in 192.168.44.{201..203}; do ssh root@${i} mkdir -p /etc/kubernetes/pki/etcd; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done #进入安装文档目录201 cd /root/k8s-ha-install/ git checkout manual-installation-v1.23.x git branch cd pki/ #生成etcd的ca证书201 cfssl gencert -initca etcd-ca-csr.json | cfssl-json -bare /etc/kubernetes/pki/etcd/etcd-ca #生成etcd的证书（注意hostname的ip地址修改预留几个可能扩容的etcd地址） cfssl gencert -ca=/etc/kubernetes/pki/etcd/etcd-ca.pem -ca-key=/etc/kubernetes/pki/etcd/etcd-ca-key.pem -config=ca-config.json -hostname=127.0.0.1,k8s-192-168-44-201.host.com,k8s-192-168-44-202.host.com,k8s-192-168-44-203.host.com,192.168.44.201,192.168.44.202,192.168.44.203,192.168.44.204,192.168.44.205,192.168.44.206,192.168.44.207 -profile=kubernetes etcd-csr.json | cfssl-json -bare /etc/kubernetes/pki/etcd/etcd #查看结果201 ll /etc/kubernetes/pki/etcd total 24 -rw-r--r-- 1 root root 1005 Dec 24 01:15 etcd-ca.csr -rw------- 1 root root 1675 Dec 24 01:15 etcd-ca-key.pem -rw-r--r-- 1 root root 1367 Dec 24 01:15 etcd-ca.pem -rw-r--r-- 1 root root 1005 Dec 24 01:40 etcd.csr -rw------- 1 root root 1675 Dec 24 01:40 etcd-key.pem -rw-r--r-- 1 root root 1562 Dec 24 01:40 etcd.pem #下发证书到其他etcd节点201 cd ~ for i in 192.168.44.{201..203}; do rsync -avzP /etc/kubernetes/pki/etcd/etcd* root@${i}:/etc/kubernetes/pki/etcd; ssh root@${i} tree /etc/kubernetes/pki/etcd; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done #验证下发结果 #202 ll /etc/kubernetes/pki/etcd total 24 -rw-r--r-- 1 root root 1005 Dec 24 01:15 etcd-ca.csr -rw------- 1 root root 1675 Dec 24 01:15 etcd-ca-key.pem -rw-r--r-- 1 root root 1367 Dec 24 01:15 etcd-ca.pem -rw-r--r-- 1 root root 1005 Dec 24 01:40 etcd.csr -rw------- 1 root root 1675 Dec 24 01:40 etcd-key.pem -rw-r--r-- 1 root root 1562 Dec 24 01:40 etcd.pem #203 ll /etc/kubernetes/pki/etcd total 24 -rw-r--r-- 1 root root 1005 Dec 24 01:15 etcd-ca.csr -rw------- 1 root root 1675 Dec 24 01:15 etcd-ca-key.pem -rw-r--r-- 1 root root 1367 Dec 24 01:15 etcd-ca.pem -rw-r--r-- 1 root root 1005 Dec 24 01:40 etcd.csr -rw------- 1 root root 1675 Dec 24 01:40 etcd-key.pem -rw-r--r-- 1 root root 1562 Dec 24 01:40 etcd.pem
2.2.2 生成k8s证书
#创建证书目录（所有master节点）201.202.203 #批量201 cd ~ for i in 192.168.44.{201..203}; do ssh root@${i} mkdir -p /etc/kubernetes/pki/etcd; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done #进入安装文档目录201 cd /root/k8s-ha-install/ git checkout manual-installation-v1.23.x git branch cd pki/ #生成ca证书201 cfssl gencert -initca ca-csr.json | cfssl-json -bare /etc/kubernetes/pki/ca #生成apiserver证书201(注意VIP地址和预留几个扩容的apiserver地址) cfssl gencert -ca=/etc/kubernetes/pki/ca.pem -ca-key=/etc/kubernetes/pki/ca-key.pem -config=ca-config.json -hostname=10.0.0.1,127.0.0.1,kubernetes,kubernetes.default,kubernetes.default.svc,kubernetes.default.svc.cluster,kubernetes.default.svc.cluster.local,192.168.44.200,192.168.44.201,192.168.44.202,192.168.44.203,192.168.44.204,192.168.44.205,192.168.44.206,192.168.44.207 -profile=kubernetes apiserver-csr.json | cfssl-json -bare /etc/kubernetes/pki/apiserver #生成聚合ca证书201 cfssl gencert -initca front-proxy-ca-csr.json | cfssl-json -bare /etc/kubernetes/pki/front-proxy-ca #生成聚合证书201 cfssl gencert -ca=/etc/kubernetes/pki/front-proxy-ca.pem -ca-key=/etc/kubernetes/pki/front-proxy-ca-key.pem -config=ca-config.json -profile=kubernetes front-proxy-client-csr.json | cfssl-json -bare /etc/kubernetes/pki/front-proxy-client #生成controller证书201 cfssl gencert -ca=/etc/kubernetes/pki/ca.pem -ca-key=/etc/kubernetes/pki/ca-key.pem -config=ca-config.json -profile=kubernetes manager-csr.json | cfssl-json -bare /etc/kubernetes/pki/controller-manager #配置controller设置201 设置集群（注意VIP地址） kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://192.168.44.200:7443 --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig 设置用户 kubectl config set-credentials system:kube-controller-manager --client-certificate=/etc/kubernetes/pki/controller-manager.pem --client-key=/etc/kubernetes/pki/controller-manager-key.pem --embed-certs=true --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig 设置上下文 kubectl config set-context system:kube-controller-manager@kubernetes --cluster=kubernetes --user=system:kube-controller-manager --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig 设置用户使用的默认上下文 kubectl config use-context system:kube-controller-manager@kubernetes --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig #生成scheduler证书201 cfssl gencert -ca=/etc/kubernetes/pki/ca.pem -ca-key=/etc/kubernetes/pki/ca-key.pem -config=ca-config.json -profile=kubernetes scheduler-csr.json | cfssl-json -bare /etc/kubernetes/pki/scheduler #配置scheduler设置201 设置集群（注意VIP地址） kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://192.168.44.200:7443 --kubeconfig=/etc/kubernetes/scheduler.kubeconfig 设置用户 kubectl config set-credentials system:kube-scheduler --client-certificate=/etc/kubernetes/pki/scheduler.pem --client-key=/etc/kubernetes/pki/scheduler-key.pem --embed-certs=true --kubeconfig=/etc/kubernetes/scheduler.kubeconfig 设置上下文 kubectl config set-context system:kube-scheduler@kubernetes --cluster=kubernetes --user=system:kube-scheduler --kubeconfig=/etc/kubernetes/scheduler.kubeconfig 设置用户使用的默认上下文 kubectl config use-context system:kube-scheduler@kubernetes --kubeconfig=/etc/kubernetes/scheduler.kubeconfig #生成admin的证书201 cfssl gencert -ca=/etc/kubernetes/pki/ca.pem -ca-key=/etc/kubernetes/pki/ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssl-json -bare /etc/kubernetes/pki/admin #配置admin的设置201 设置集群（注意VIP地址） kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://192.168.44.200:7443 --kubeconfig=/etc/kubernetes/admin.kubeconfig 设置用户 kubectl config set-credentials kubernetes-admin --client-certificate=/etc/kubernetes/pki/admin.pem --client-key=/etc/kubernetes/pki/admin-key.pem --embed-certs=true --kubeconfig=/etc/kubernetes/admin.kubeconfig 设置上下文 kubectl config set-context kubernetes-admin@kubernetes --cluster=kubernetes --user=kubernetes-admin --kubeconfig=/etc/kubernetes/admin.kubeconfig 设置默认上下文 kubectl config use-context kubernetes-admin@kubernetes --kubeconfig=/etc/kubernetes/admin.kubeconfig #创建ServiceAccount Key 201 openssl genrsa -out /etc/kubernetes/pki/sa.key 2048 openssl rsa -in /etc/kubernetes/pki/sa.key -pubout -out /etc/kubernetes/pki/sa.pub #查看结果201 tree /etc/kubernetes/ /etc/kubernetes/ ├── admin.kubeconfig ├── controller-manager.kubeconfig ├── pki │ ├── admin.csr │ ├── admin-key.pem │ ├── admin.pem │ ├── apiserver.csr │ ├── apiserver-key.pem │ ├── apiserver.pem │ ├── ca.csr │ ├── ca-key.pem │ ├── ca.pem │ ├── controller-manager.csr │ ├── controller-manager-key.pem │ ├── controller-manager.pem │ ├── etcd │ │ ├── etcd-ca.csr │ │ ├── etcd-ca-key.pem │ │ ├── etcd-ca.pem │ │ ├── etcd.csr │ │ ├── etcd-key.pem │ │ └── etcd.pem │ ├── front-proxy-ca.csr │ ├── front-proxy-ca-key.pem │ ├── front-proxy-ca.pem │ ├── front-proxy-client.csr │ ├── front-proxy-client-key.pem │ ├── front-proxy-client.pem │ ├── sa.key │ ├── sa.pub │ ├── scheduler.csr │ ├── scheduler-key.pem │ └── scheduler.pem └── scheduler.kubeconfig 2 directories, 32 files #下发证书到其他master节点201 cd ~ for i in 192.168.44.{201..203}; do rsync -avzP /etc/kubernetes/ root@${i}:/etc/kubernetes; ssh root@${i} tree /etc/kubernetes/; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done #查看结果 #202 tree /etc/kubernetes/ /etc/kubernetes/ ├── admin.kubeconfig ├── controller-manager.kubeconfig ├── pki │ ├── admin.csr │ ├── admin-key.pem │ ├── admin.pem │ ├── apiserver.csr │ ├── apiserver-key.pem │ ├── apiserver.pem │ ├── ca.csr │ ├── ca-key.pem │ ├── ca.pem │ ├── controller-manager.csr │ ├── controller-manager-key.pem │ ├── controller-manager.pem │ ├── etcd │ │ ├── etcd-ca.csr │ │ ├── etcd-ca-key.pem │ │ ├── etcd-ca.pem │ │ ├── etcd.csr │ │ ├── etcd-key.pem │ │ └── etcd.pem │ ├── front-proxy-ca.csr │ ├── front-proxy-ca-key.pem │ ├── front-proxy-ca.pem │ ├── front-proxy-client.csr │ ├── front-proxy-client-key.pem │ ├── front-proxy-client.pem │ ├── sa.key │ ├── sa.pub │ ├── scheduler.csr │ ├── scheduler-key.pem │ └── scheduler.pem └── scheduler.kubeconfig 2 directories, 32 files #203 tree /etc/kubernetes/ /etc/kubernetes/ ├── admin.kubeconfig ├── controller-manager.kubeconfig ├── pki │ ├── admin.csr │ ├── admin-key.pem │ ├── admin.pem │ ├── apiserver.csr │ ├── apiserver-key.pem │ ├── apiserver.pem │ ├── ca.csr │ ├── ca-key.pem │ ├── ca.pem │ ├── controller-manager.csr │ ├── controller-manager-key.pem │ ├── controller-manager.pem │ ├── etcd │ │ ├── etcd-ca.csr │ │ ├── etcd-ca-key.pem │ │ ├── etcd-ca.pem │ │ ├── etcd.csr │ │ ├── etcd-key.pem │ │ └── etcd.pem │ ├── front-proxy-ca.csr │ ├── front-proxy-ca-key.pem │ ├── front-proxy-ca.pem │ ├── front-proxy-client.csr │ ├── front-proxy-client-key.pem │ ├── front-proxy-client.pem │ ├── sa.key │ ├── sa.pub │ ├── scheduler.csr │ ├── scheduler-key.pem │ └── scheduler.pem └── scheduler.kubeconfig 2 directories, 32 files2.3 安装etcd
准备配置文件
#注意修改ip地址 #201 mkdir /etc/etcd -p vim /etc/etcd/etcd.conf #[Member] ETCD_NAME="etcd1" ETCD_DATA_DIR="/var/lib/etcd" ETCD_LISTEN_PEER_URLS="https://192.168.44.201:2380" ETCD_LISTEN_CLIENT_URLS="https://192.168.44.201:2379,http://127.0.0.1:2379" #[Clustering] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.44.201:2380" ETCD_ADVERTISE_CLIENT_URLS="https://192.168.44.201:2379" ETCD_INITIAL_CLUSTER="etcd1=https://192.168.44.201:2380,etcd2=https://192.168.44.202:2380,etcd3=https://192.168.44.203:2380" ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" ETCD_INITIAL_CLUSTER_STATE="new" #202 mkdir /etc/etcd -p vim /etc/etcd/etcd.conf #[Member] ETCD_NAME="etcd2" ETCD_DATA_DIR="/var/lib/etcd" ETCD_LISTEN_PEER_URLS="https://192.168.44.202:2380" ETCD_LISTEN_CLIENT_URLS="https://192.168.44.202:2379,http://127.0.0.1:2379" #[Clustering] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.44.202:2380" ETCD_ADVERTISE_CLIENT_URLS="https://192.168.44.202:2379" ETCD_INITIAL_CLUSTER="etcd1=https://192.168.44.201:2380,etcd2=https://192.168.44.202:2380,etcd3=https://192.168.44.203:2380" ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" ETCD_INITIAL_CLUSTER_STATE="new" #203 mkdir /etc/etcd -p vim /etc/etcd/etcd.conf #[Member] ETCD_NAME="etcd3" ETCD_DATA_DIR="/var/lib/etcd" ETCD_LISTEN_PEER_URLS="https://192.168.44.203:2380" ETCD_LISTEN_CLIENT_URLS="https://192.168.44.203:2379,http://127.0.0.1:2379" #[Clustering] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.44.203:2380" ETCD_ADVERTISE_CLIENT_URLS="https://192.168.44.203:2379" ETCD_INITIAL_CLUSTER="etcd1=https://192.168.44.201:2380,etcd2=https://192.168.44.202:2380,etcd3=https://192.168.44.203:2380" ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" ETCD_INITIAL_CLUSTER_STATE="new"准备启动文件
#所有etcd节点201.202.203 #单机编辑201 cat >/usr/lib/systemd/system/etcd.service <2.4 安装apisercer
#创建相关目录（所有master节点）201.202.203 #批量201 cd ~ for i in 192.168.44.{201..203}; do ssh root@${i} mkdir -p /opt/cni/bin /etc/cni/net.d /etc/kubernetes/manifests /etc/systemd/system/kubelet.service.d /var/lib/kubelet /var/log/kubernetes; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done #创建token文件（所有master节点）201.202.203 #单机编辑201 cat >/etc/kubernetes/token.csv </usr/lib/systemd/system/kube-apiserver.service < /usr/lib/systemd/system/kube-apiserver.service < /usr/lib/systemd/system/kube-apiserver.service < 2.5 安装controller
#准备启动文件(注意修改pod的ip地址段，本次10.244.0.0/16） #注意使用相同版本的虚拟机，打开kubeadm集群查看对应参数，必要时调整 #所有master节点201.202.203 cat >/usr/lib/systemd/system/kube-controller-manager.service <2.6 安装scheduler
#准备启动文件 #注意使用相同版本的虚拟机，打开kubeadm集群查看对应参数，必要时调整 #所有master节点201.202.203 cat >/usr/lib/systemd/system/kube-scheduler.service <第3章 k8s安装集群node
3.1 TLS Bootstrapping配置
官方文档：https://kubernetes.io/zh/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/#bootstrap-tokens
#配置Bootstrapping配置文件201 cd /root/k8s-ha-install/ git checkout manual-installation-v1.23.x git branch cd bootstrap/ 创建集群(注意VIP地址) kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://192.168.44.200:7443 --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig 创建用户（注意token要和bootstrap.secret.yaml或者token.csv文件里边一样） kubectl config set-credentials tls-bootstrap-token-user --token=c8ad9c.2e4d610cf3e7426e --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig 创建上下文 kubectl config set-context tls-bootstrap-token-user@kubernetes --cluster=kubernetes --user=tls-bootstrap-token-user --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig 创建默认上下文 kubectl config use-context tls-bootstrap-token-user@kubernetes --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig #所有master节点201.202.203 mkdir /root/.kube -p cp /etc/kubernetes/admin.kubeconfig /root/.kube/config #配置token以及RBAC规则201 ============================================= 方法一：令牌认证文件（推荐，因为token永不过期） cat /etc/kubernetes/token.csv 02b50b05283e98dd0fd71db496ef01e8,kubelet-bootstrap,10001,"system:bootstrappers" #需要api开启这个特性 --token-auth-file=/etc/kubernetes/token.csv ============================================= 方法二：启动引导令牌（推荐，因为token过期会自动被清理） #需要api组件开启这个特性 --enable-bootstrap-token-auth=true ============================================== cat ~/k8s-ha-install/bootstrap/bootstrap.secret.yaml #启动令牌认证 apiVersion: v1 kind: Secret metadata: # name 必须是 "bootstrap-token-" 格式的 name: bootstrap-token-c8ad9c namespace: kube-system # type 必须是 'bootstrap.kubernetes.io/token' type: bootstrap.kubernetes.io/token stringData: # 供人阅读的描述，可选。 description: "The default bootstrap token generated by 'kubelet '." # 令牌 ID 和秘密信息，必需。 token-id: c8ad9c token-secret: 2e4d610cf3e7426e # 可选的过期时间字段 expiration: "2999-03-10T03:22:11Z" # 允许的用法 usage-bootstrap-authentication: "true" usage-bootstrap-signing: "true" # 令牌要认证为的额外组，必须以 "system:bootstrappers:" 开头 auth-extra-groups: system:bootstrappers:default-node-token,system:bootstrappers:worker,system:bootstrappers:ingress --- # 允许启动引导节点创建 CSR apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: create-csrs-for-bootstrapping subjects: - kind: Group name: system:bootstrappers apiGroup: rbac.authorization.k8s.io roleRef: kind: ClusterRole name: system:node-bootstrapper apiGroup: rbac.authorization.k8s.io --- # 批复 "system:bootstrappers" 组的所有 客户端CSR apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: auto-approve-csrs-for-group subjects: - kind: Group name: system:bootstrappers apiGroup: rbac.authorization.k8s.io roleRef: kind: ClusterRole name: system:certificates.k8s.io:certificatesigningrequests:nodeclient apiGroup: rbac.authorization.k8s.io --- # 批复 "system:nodes" 组的 客户端 CSR 续约请求 apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: auto-approve-renewals-for-nodes subjects: - kind: Group name: system:nodes apiGroup: rbac.authorization.k8s.io roleRef: kind: ClusterRole name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient apiGroup: rbac.authorization.k8s.io --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: annotations: rbac.authorization.kubernetes.io/autoupdate: "true" labels: kubernetes.io/bootstrapping: rbac-defaults name: system:kube-apiserver-to-kubelet rules: - apiGroups: - "" resources: - nodes/proxy - nodes/stats - nodes/log - nodes/spec - nodes/metrics verbs: - "*" --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: system:kube-apiserver namespace: "" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:kube-apiserver-to-kubelet subjects: - apiGroup: rbac.authorization.k8s.io kind: User name: kube-apiserver =========================================================== #应用资源201 kubectl create -f bootstrap.secret.yaml 3.2 node证书配置
#创建证书目录（所有node节点）201.202.203 #批量201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do ssh root@${i} mkdir -p /etc/kubernetes/pki/etcd; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done #创建证书目录（所有node节点）201.202.203 #批量201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do ssh root@${i} mkdir -p /etc/kubernetes/pki; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done #下发证书201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do rsync -avzP /etc/kubernetes/ --exclude=kubelet.kubeconfig root@${i}:/etc/kubernetes; ssh root@${i} tree /etc/kubernetes/; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done #检查 #202 tree /etc/kubernetes/ /etc/kubernetes/ ├── admin.kubeconfig ├── bootstrap-kubelet.kubeconfig ├── controller-manager.kubeconfig ├── manifests ├── pki │ ├── admin.csr │ ├── admin-key.pem │ ├── admin.pem │ ├── apiserver.csr │ ├── apiserver-key.pem │ ├── apiserver.pem │ ├── ca.csr │ ├── ca-key.pem │ ├── ca.pem │ ├── controller-manager.csr │ ├── controller-manager-key.pem │ ├── controller-manager.pem │ ├── etcd │ │ ├── etcd-ca.csr │ │ ├── etcd-ca-key.pem │ │ ├── etcd-ca.pem │ │ ├── etcd.csr │ │ ├── etcd-key.pem │ │ └── etcd.pem │ ├── front-proxy-ca.csr │ ├── front-proxy-ca-key.pem │ ├── front-proxy-ca.pem │ ├── front-proxy-client.csr │ ├── front-proxy-client-key.pem │ ├── front-proxy-client.pem │ ├── sa.key │ ├── sa.pub │ ├── scheduler.csr │ ├── scheduler-key.pem │ └── scheduler.pem └── scheduler.kubeconfig └── token.csv 3 directories, 34 files3.3 安装kubelet
#所有node节点创建相关目录201.202.203 #批量201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do ssh root@${i} mkdir -p /opt/cni/bin /etc/cni/net.d /var/lib/kubelet /var/log/kubernetes /etc/systemd/system/kubelet.service.d /etc/kubernetes/manifests/; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done #所有node节点配置启动文件201.202.203 #注意使用相同版本的虚拟机，打开kubeadm集群查看对应参数，必要时调整 #单机编辑201 cat >/usr/lib/systemd/system/kubelet.service </etc/systemd/system/kubelet.service.d/10-kubelet.conf < /etc/kubernetes/kubelet-conf.yml < lines 1114-1141/1141 (END) #查看证书有效期201.202.203 #批量201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do ssh root@${i} openssl x509 -in /var/lib/kubelet/pki/kubelet-client-current.pem -noout -text |grep -A 1 "Not Before:"; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done 3.4 安装kube-proxy
#创建配置201（注意vip地址） cd /root/k8s-ha-install/ git branch #1 kubectl -n kube-system create serviceaccount kube-proxy #2 kubectl create clusterrolebinding system:kube-proxy --clusterrole system:node-proxier --serviceaccount kube-system:kube-proxy #3 SECRET=$(kubectl -n kube-system get sa/kube-proxy --output=jsonpath='{.secrets[0].name}') #4 JWT_TOKEN=$(kubectl -n kube-system get secret/$SECRET --output=jsonpath='{.data.token}' | base64 -d) #5 PKI_DIR=/etc/kubernetes/pki #6 K8S_DIR=/etc/kubernetes #7 kubectl config set-cluster kubernetes --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://192.168.44.200:7443 --kubeconfig=${K8S_DIR}/kube-proxy.kubeconfig #8 kubectl config set-credentials kubernetes --token=${JWT_TOKEN} --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig #9 kubectl config set-context kubernetes --cluster=kubernetes --user=kubernetes --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig #10 kubectl config use-context kubernetes --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig #分发配置到其他node节点201 cd /root/k8s-ha-install/ git branch ll kube-proxy/ total 12 -rw-r--r-- 1 root root 813 Dec 24 01:06 kube-proxy.conf -rw-r--r-- 1 root root 288 Dec 24 01:06 kube-proxy.service -rw-r--r-- 1 root root 3677 Dec 24 01:06 kube-proxy.yml #编辑kube-proxy.conf文件修改pod网络10.244.0.0/16 #注意使用相同版本的虚拟机，打开kubeadm集群查看对应参数，必要时调整 #单机编辑201 cat >/etc/kubernetes/kube-proxy.yml </usr/lib/systemd/system/kube-proxy.service < 第4章 k8s安装集群插件
4.1 安装calico
https://kubernetes.io/docs/concepts/cluster-administration/addons/ #201 这里calico cd /root/k8s-ha-install git checkout manual-installation-v1.23.x git branch cd calico/ #修改文件内容，注意ip地址201 cp calico-etcd.yaml calico-etcd.yaml.bak$(date +"%F-%H-%M-%S") sed -i 's#etcd_endpoints: "http://: "#etcd_endpoints: "https://192.168.44.201:2379,https://192.168.44.202:2379,https://192.168.44.203:2379"#g' calico-etcd.yaml ETCD_CA=`cat /etc/kubernetes/pki/etcd/etcd-ca.pem | base64 | tr -d 'n'` ETCD_CERT=`cat /etc/kubernetes/pki/etcd/etcd.pem | base64 | tr -d 'n'` ETCD_KEY=`cat /etc/kubernetes/pki/etcd/etcd-key.pem | base64 | tr -d 'n'` sed -i "s@# etcd-key: null@etcd-key: ${ETCD_KEY}@g; s@# etcd-cert: null@etcd-cert: ${ETCD_CERT}@g; s@# etcd-ca: null@etcd-ca: ${ETCD_CA}@g" calico-etcd.yaml sed -i 's#etcd_ca: ""#etcd_ca: "/calico-secrets/etcd-ca"#g; s#etcd_cert: ""#etcd_cert: "/calico-secrets/etcd-cert"#g; s#etcd_key: "" #etcd_key: "/calico-secrets/etcd-key" #g' calico-etcd.yaml POD_SUBNET="10.244.0.0/16" sed -i 's@- name: CALICO_IPV4POOL_CIDR@- name: CALICO_IPV4POOL_CIDR@g; s@value: "POD_CIDR"@value: '"${POD_SUBNET}"'@g' calico-etcd.yaml #安装201 kubectl apply -f calico-etcd.yaml kubectl get --all-namespaces pod kubectl get service NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE kubernetes ClusterIP 10.0.0.1 443/TCP 77m kubectl get nodes NAME STATUS ROLES AGE VERSION k8s-192-168-44-201.host.com Ready 33m v1.23.5 k8s-192-168-44-202.host.com Ready 33m v1.23.5 k8s-192-168-44-203.host.com Ready 33m v1.23.5 #修改节点角色（所有master节点）201 cd ~ for i in k8s-192-168-44-{201..203}.host.com; do kubectl label node ${i} node-role.kubernetes.io/master=; kubectl label node ${i} node-role.kubernetes.io/node=; done #修改节点角色（所有node节点）201 cd ~ for i in $(kubectl get nodes |awk "NR>1"'{print $1}'); do kubectl label node ${i} node-role.kubernetes.io/node=; done #查看效果201 kubectl get nodes NAME STATUS ROLES AGE VERSION k8s-192-168-44-201.host.com Ready master,node 37m v1.23.5 k8s-192-168-44-202.host.com Ready master,node 37m v1.23.5 k8s-192-168-44-203.host.com Ready master,node 37m v1.23.5 4.2 安装coredns
#201 cd /root/k8s-ha-install/ git checkout manual-installation-v1.23.x git branch cd CoreDNS/ #修改coredns的IP地址10.0.0.10 vim coredns.yaml spec: selector: k8s-app: kube-dns clusterIP: 10.0.0.10 #应用资源清单201 kubectl apply -f coredns.yaml #查看结果 kubectl get --all-namespaces all NAMESPACE NAME READY STATUS RESTARTS AGE kube-system pod/calico-kube-controllers-5f6d4b864b-rfgxl 1/1 Running 0 55m kube-system pod/calico-node-4f7h9 1/1 Running 0 55m kube-system pod/calico-node-4j57l 1/1 Running 0 55m kube-system pod/calico-node-r8dxd 1/1 Running 0 55m kube-system pod/coredns-867d46bfc6-9zt6d 1/1 Running 0 23s NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE default service/kubernetes ClusterIP 10.0.0.1443/TCP 10h kube-system service/kube-dns ClusterIP 10.0.0.10 53/UDP,53/TCP,9153/TCP 24s NAMESPACE NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE kube-system daemonset.apps/calico-node 3 3 3 3 3 kubernetes.io/os=linux 55m NAMESPACE NAME READY UP-TO-DATE AVAILABLE AGE kube-system deployment.apps/calico-kube-controllers 1/1 1 1 55m kube-system deployment.apps/coredns 1/1 1 1 25s NAMESPACE NAME DESIRED CURRENT READY AGE kube-system replicaset.apps/calico-kube-controllers-5f6d4b864b 1 1 1 55m kube-system replicaset.apps/coredns-867d46bfc6 1 1 1 24s 4.3 安装metrics-server
#201 cd /root/k8s-ha-install/metrics-server/ kubectl apply -f comp.yaml kubectl get --all-namespaces pod kubectl top node kubectl top pod --all-namespaces #镜像批量保存脚本 cat image_save.sh #!/bin/bash #fun: save docker image list to local disk source /etc/init.d/functions docker image ls|awk 'NR>1''{print $1":"$2}' >/tmp/image_list.txt for image_name in $(cat /tmp/image_list.txt) do tmp_name=${image_name-} tmp1_name=${tmp_name//./-} file_name=${tmp1_name//:/-}.tar echo "saveing image ${image_name}...." docker image save ${image_name} >/tmp/${file_name} action "image ${image_name} has saved complete!" /bin/true done #镜像批量导入脚本 cat image_load.sh #!/bin/bash #fun: load docker image list from local disk source /etc/init.d/functions for image_name in $(find /tmp/ -name "*.tar") do echo "loading image ${image_name///tmp//}...." docker image load -i ${image_name} action "image ${image_name///tmp//} has loaded complete!" /bin/true done docker image ls echo "导入完成！本次导入了 $(find /tmp/ -name "*.tar"|wc -l) 个镜像，系统共计 $(docker image ls |awk 'NR>1'|wc -l) 个镜像。"4.4 安装ingress-nginx
可以略过，参考第9章节ingress资源详解来安装最新版！！！！
#201 地址：https://github.com/kubernetes/ingress-nginx mkdir /data/k8s-yaml/ingress-controller -p cd /data/k8s-yaml/ingress-controller ===================================================== cat ingress-controller.yaml apiVersion: v1 kind: Namespace metadata: name: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx --- kind: ConfigMap apiVersion: v1 metadata: name: nginx-configuration namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx --- kind: ConfigMap apiVersion: v1 metadata: name: tcp-services namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx --- kind: ConfigMap apiVersion: v1 metadata: name: udp-services namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx --- apiVersion: v1 kind: ServiceAccount metadata: name: nginx-ingress-serviceaccount namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: nginx-ingress-clusterrole labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx rules: - apiGroups: - "" resources: - configmaps - endpoints - nodes - pods - secrets verbs: - list - watch - apiGroups: - "" resources: - nodes verbs: - get - apiGroups: - "" resources: - services verbs: - get - list - watch - apiGroups: - "" resources: - events verbs: - create - patch - apiGroups: - "extensions" - "networking.k8s.io" resources: - ingresses verbs: - get - list - watch - apiGroups: - "extensions" - "networking.k8s.io" resources: - ingresses/status verbs: - update --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: Role metadata: name: nginx-ingress-role namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx rules: - apiGroups: - "" resources: - configmaps - pods - secrets - namespaces verbs: - get - apiGroups: - "" resources: - configmaps resourceNames: # Defaults to "- " # Here: " - " # This has to be adapted if you change either parameter # when launching the nginx-ingress-controller. - "ingress-controller-leader-nginx" verbs: - get - update - apiGroups: - "" resources: - configmaps verbs: - create - apiGroups: - "" resources: - endpoints verbs: - get --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: RoleBinding metadata: name: nginx-ingress-role-nisa-binding namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: nginx-ingress-role subjects: - kind: ServiceAccount name: nginx-ingress-serviceaccount namespace: ingress-nginx --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: name: nginx-ingress-clusterrole-nisa-binding labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: nginx-ingress-clusterrole subjects: - kind: ServiceAccount name: nginx-ingress-serviceaccount namespace: ingress-nginx --- apiVersion: apps/v1 kind: Deployment metadata: name: nginx-ingress-controller namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx spec: replicas: 1 selector: matchLabels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx template: metadata: labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx annotations: prometheus.io/port: "10254" prometheus.io/scrape: "true" spec: #hostNetwork: true # wait up to five minutes for the drain of connections terminationGracePeriodSeconds: 300 serviceAccountName: nginx-ingress-serviceaccount nodeSelector: kubernetes.io/os: linux containers: - name: nginx-ingress-controller image: lizhenliang/nginx-ingress-controller:0.30.0 args: - /nginx-ingress-controller - --configmap=$(POD_NAMESPACE)/nginx-configuration - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services - --udp-services-configmap=$(POD_NAMESPACE)/udp-services - --publish-service=$(POD_NAMESPACE)/ingress-nginx - --annotations-prefix=nginx.ingress.kubernetes.io securityContext: allowPrivilegeEscalation: true capabilities: drop: - ALL add: - NET_BIND_SERVICE # www-data -> 101 runAsUser: 101 env: - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace ports: - name: http containerPort: 80 protocol: TCP - name: https containerPort: 443 protocol: TCP livenessProbe: failureThreshold: 3 httpGet: path: /healthz port: 10254 scheme: HTTP initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 10 readinessProbe: failureThreshold: 3 httpGet: path: /healthz port: 10254 scheme: HTTP periodSeconds: 10 successThreshold: 1 timeoutSeconds: 10 lifecycle: preStop: exec: command: - /wait-shutdown --- apiVersion: v1 kind: LimitRange metadata: name: ingress-nginx namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx spec: limits: - min: memory: 90Mi cpu: 100m type: Container ===================================================== cat svc.yaml apiVersion: v1 kind: Service metadata: creationTimestamp: null labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx name: nginx-ingress-controller namespace: ingress-nginx spec: ports: - name: port-1 port: 80 protocol: TCP targetPort: 80 nodePort: 30030 - name: port-2 port: 443 protocol: TCP targetPort: 443 nodePort: 30033 selector: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx type: NodePort status: loadBalancer: {} ==================================================== #应用资源201 kubectl apply -f ingress-controller.yaml kubectl apply -f svc.yaml kubectl get -n ingress-nginx all NAME READY STATUS RESTARTS AGE pod/nginx-ingress-controller-5759f86b89-bmx9q 1/1 Running 0 4m2s NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE service/nginx-ingress-controller NodePort 10.0.158.23 80:30030/TCP,443:30033/TCP 4m NAME READY UP-TO-DATE AVAILABLE AGE deployment.apps/nginx-ingress-controller 1/1 1 1 4m2s NAME DESIRED CURRENT READY AGE replicaset.apps/nginx-ingress-controller-5759f86b89 1 1 1 4m2s #配置代理201.202.203 cat >/etc/nginx/conf.d/od.com.conf < 4.5 配置storageClass动态存储
#服务端201 mkdir /volume -p yum install -y nfs-utils rpcbind echo '/volume 192.168.44.0/24(insecure,rw,sync,no_root_squash,no_all_squash)' >/etc/exports systemctl start rpcbind.service systemctl enable rpcbind.service systemctl start nfs-server systemctl enable nfs-server #客户端所有node节点202 && 203 #批量201 cd ~ for i in $(cat host_list.txt |awk '{print $1}'); do ssh root@${i} yum install -y nfs-utils; ssh root@${i} showmount -e 192.168.44.201; ssh root@${i} echo "==================================================="; ssh root@${i} echo "${i}"; ssh root@${i} echo "==================================================="; ssh root@${i} echo; done #创建目录（第1个存储类就创建对应编号1的目录）201 mkdir /data/k8s-yaml/storageClass/1 -p cd /data/k8s-yaml/storageClass/1 ================================================== cat >rbac.yaml <dp.yaml < sc.yaml < 4.6 安装kubesphere集群管理
https://kubesphere.com.cn/
#201 mkdir /data/k8s-yaml/kubesphere -p cd /data/k8s-yaml/kubesphere/ wget https://github.com/kubesphere/ks-installer/releases/download/v3.2.1/kubesphere-installer.yaml wget https://github.com/kubesphere/ks-installer/releases/download/v3.2.1/cluster-configuration.yaml #修改配置 vim cluster-configuration.yaml etcd: monitoring: true # Enable or disable etcd monitoring dashboard installation. You have to create a Secret for etcd before you enable it. endpointIps: 192.168.44.201 # etcd cluster EndpointIps. It can be a bunch of IPs here. port: 2379 # etcd port. tlsEnable: true redis: enabled: true openldap: enabled: true basicAuth: enabled: false console: enableMultiLogin: true alerting: # (CPU: 0.1 Core, Memory: 100 MiB) It enables users to customize alerting policies to send messages to receivers in time with different time intervals and alerting levels to choose from. enabled: true auditing: # Provide a security-relevant chronological set of records，recording the sequence of activities happening on the platform, initiated by different tenants. enabled: true devops: # (CPU: 0.47 Core, Memory: 8.6 G) Provide an out-of-the-box CI/CD system based on Jenkins, and automated workflow tools including Source-to-Image & Binary-to-Image. enabled: true events: # Provide a graphical web console for Kubernetes Events exporting, filtering and alerting in multi-tenant Kubernetes clusters. enabled: true ruler: enabled: true logging: # (CPU: 57 m, Memory: 2.76 G) Flexible logging functions are provided for log query, collection and management in a unified console. Additional log collectors can be added, such as Elasticsearch, Kafka and Fluentd. enabled: true logsidecar: enabled: true metrics_server: # (CPU: 56 m, Memory: 44.35 MiB) It enables HPA (Horizontal Pod Autoscaler). enabled: false networkpolicy: # Network policies allow network isolation within the same cluster, which means firewalls can be set up between certain instances (Pods). # Make sure that the CNI network plugin used by the cluster supports NetworkPolicy. There are a number of CNI network plugins that support NetworkPolicy, including Calico, Cilium, Kube-router, Romana and Weave Net. enabled: true store: enabled: true servicemesh: # (0.3 Core, 300 MiB) Provide fine-grained traffic management, observability and tracing, and visualized traffic topology. enabled: true kubeedge: # Add edge nodes to your cluster and deploy workloads on edge nodes. enabled: true #将master设置为可调度，因为这平台耗费资源厉害 kubectl taint node k8s-192-168-44-201.host.com node-role.kubernetes.io/master- kubectl describe node k8s-192-168-44-201.host.com #将节点打标签 kubectl label nodes k8s-192-168-44-201.host.com node-role.kubernetes.io/woker= #应用资源 kubectl apply -f kubesphere-installer.yaml kubectl apply -f cluster-configuration.yaml #查看效果 kubectl get --all-namespaces pod kubesphere-system ks-installer-54c6bcf76b-whxgj 0/1 ContainerCreating 0 74s #查看安装日志不允许出现失败的 kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l app=ks-install -o jsonpath='{.items[0].metadata.name}') -f localhost : ok=31 changed=25 unreachable=0 failed=0 skipped=15 rescued=0 ignored=0 #安装完成 localhost : ok=31 changed=25 unreachable=0 failed=0 skipped=15 rescued=0 ignored=0 Start installing monitoring Start installing multicluster Start installing openpitrix Start installing network ************************************************** Waiting for all tasks to be completed ... task network status is successful (1/4) task openpitrix status is successful (2/4) task multicluster status is successful (3/4) task monitoring status is successful (4/4) ************************************************** Collecting installation results ... ##################################################### ### Welcome to KubeSphere! ### ##################################################### Console: http://192.168.44.201:30880 Account: admin Password: P@88w0rd NOTES： 1. After you log into the console, please check the monitoring status of service components in "Cluster Management". If any service is not ready, please wait patiently until all components are up and running. 2. Please change the default password after login. ##################################################### https://kubesphere.io 2021-10-03 19:21:41 #####################################################安装完成界面效果
监控模块报错解决
#201 kubectl -n kubesphere-monitoring-system create secret generic kube-etcd-client-certs --from-file=etcd-client-ca.crt=/etc/kubernetes/pki/etcd/etcd-ca.pem --from-file=etcd-client.crt=/etc/kubernetes/pki/etcd/etcd.pem --from-file=etcd-client.key=/etc/kubernetes/pki/etcd/etcd-key.pem
