1、环境准备
要点： 1、使用一个FQDN统一作为API Server的接入点； 2、加入集群之前，每个节点都将该FQDN解析至第一个Master； 3、加入集群之后，每个Master节点将该FQDN都解析至自身的IP地址； 4、加入集群之后，在Worker上配置nginx以对API Server进行代理，并将该FQDN解析至自身的IP地址；
1-1、主机清单
| 主机名 | IP地址 | 系统版本 |
|---|---|---|
| k8s-master01 k8s-master01.wang.org kubeapi.wang.org kubeapi | 10.0.0.101 | Ubuntu2004 |
| k8s-master02 k8s-master02.wang.org | 10.0.0.102 | Ubuntu2004 |
| k8s-master03 k8s-master03.wang.org | 10.0.0.103 | Ubuntu2004 |
| k8s-node01 k8s-node01.wang.org | 10.0.0.104 | Ubuntu2004 |
| k8s-node02 k8s-node02.wang.org | 10.0.0.105 | Ubuntu2004 |
1-1、设置主机名
#所有节点执行： [root@ubuntu2004 ~]#hostnamectl set-hostname k8s-master011-2、关闭防火墙
#所有节点执行： [root@k8s-master01 ~]# ufw disable [root@k8s-master01 ~]# ufw status1-3、时间同步
#所有节点执行： [root@k8s-master01 ~]# apt install -y chrony [root@k8s-master01 ~]# systemctl restart chrony [root@k8s-master01 ~]# systemctl status chrony [root@k8s-master01 ~]# chronyc sources1-4、主机名互相解析
#所有节点执行： [root@k8s-master01 ~]#vim /etc/hosts 10.0.0.101 k8s-master01 k8s-master01.wang.org kubeapi.wang.org kubeapi 10.0.0.102 k8s-master02 k8s-master02.wang.org 10.0.0.103 k8s-master03 k8s-master03.wang.org 10.0.0.104 k8s-node01 k8s-node01.wang.org 10.0.0.105 k8s-node02 k8s-node02.wang.org1-5、禁用swap
#所有节点执行： [root@k8s-master01 ~]# sed -r -i '//swap/s@^@#@' /etc/fstab [root@k8s-master01 ~]# swapoff -a [root@k8s-master01 ~]# systemctl --type swap #若不禁用Swap设备，需要在后续编辑kubelet的配置文件/etc/default/kubelet，设置其忽略Swap启用的状态错误，内容：KUBELET_EXTRA_ARGS="--fail-swap-on=false"2、安装docker
#所有节点执行： #安装必要的一些系统工具 [root@k8s-master01 ~]# apt update [root@k8s-master01 ~]# apt -y install apt-transport-https ca-certificates curl software-properties-common #安装GPG证书 [root@k8s-master01 ~]# curl -fsSL http://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | apt-key add - OK #写入软件源信息 [root@k8s-master01 ~]# add-apt-repository "deb [arch=amd64] http://mirrors.aliyun.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable" #更新并安装Docker-CE [root@k8s-master01 ~]# apt update [root@k8s-master01 ~]# apt install -y docker-ce #所有节点执行： kubelet需要让docker容器引擎使用systemd作为CGroup的驱动，其默认值为cgroupfs，因而，我们还需要编辑docker的配置文件/etc/docker/daemon.json，添加如下内容，其中的registry-mirrors用于指明使用的镜像加速服务。 [root@k8s-master01 ~]# mkdir /etc/docker #如果已存在，请忽略此步 [root@k8s-master01 ~]# vim /etc/docker/daemon.json { "registry-mirrors": [ "https://docker.mirrors.ustc.edu.cn", "https://hub-mirror.c.163.com", "https://reg-mirror.qiniu.com", "https://registry.docker-cn.com", "https://pgavrk5n.mirror.aliyuncs.com" ], "exec-opts": ["native.cgroupdriver=systemd"], "log-driver": "json-file", "log-opts": { "max-size": "200m" }, "storage-driver": "overlay2" } [root@k8s-master01 ~]#systemctl daemon-reload && systemctl enable --now docker && docker version Client: Docker Engine - Community Version: 20.10.21 #注：kubeadm部署Kubernetes集群的过程中，默认使用Google的Registry服务k8s.gcr.io上的镜像,由于2022年仓库已经改为registry.k8s.io，国内可以直接访问，所以现在不需要镜像加速或者绿色上网就可以拉镜像了，如果使用国内镜像请参考https://blog.51cto.com/dayu/58113073、安装cri-dockerd
#所有节点执行： #下载地址：https://github.com/Mirantis/cri-dockerd [root@k8s-master01 ~]# apt install ./cri-dockerd_0.2.6.3-0.ubuntu-focal_amd64.deb -y #完成安装后，相应的服务cri-dockerd.service便会自动启动 [root@k8s-master01 ~]#systemctl restart cri-docker.service && systemctl status cri-docker.service4、安装kubeadm、kubelet和kubectl
#所有节点执行： #在各主机上生成kubelet和kubeadm等相关程序包的仓库，可参考阿里云官网 [root@k8s-master01 ~]# apt update [root@k8s-master01 ~]# apt install -y apt-transport-https curl [root@k8s-master01 ~]# curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add - [root@k8s-master01 ~]#cat <<EOF >/etc/apt/sources.list.d/kubernetes.list deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main EOF #更新仓库并安装 [root@k8s-master01 ~]# apt update [root@k8s-master01 ~]# apt install -y kubelet kubeadm kubectl #注意：先不要启动，只是设置开机自启动 [root@k8s-master01 ~]# systemctl enable kubelet #确定kubeadm等程序文件的版本 [root@k8s-master01 ~]# kubeadm version kubeadm version: &version.Info{Major:"1", Minor:"25", GitVersion:"v1.25.4", GitCommit:"872a965c6c6526caa949f0c6ac028ef7aff3fb78", GitTreeState:"clean", BuildDate:"2022-11-09T13:35:06Z", GoVersion:"go1.19.3", Compiler:"gc", Platform:"linux/amd64"}5、整合kubelet和cri-dockerd
5-1、配置cri-dockerd
#所有节点执行： [root@k8s-master01 ~]# vim /usr/lib/systemd/system/cri-docker.service #ExecStart=/usr/bin/cri-dockerd --container-runtime-endpoint fd:// ExecStart=/usr/bin/cri-dockerd --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.8 --container-runtime-endpoint fd:// --network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-cache-dir=/var/lib/cni/cache --cni-conf-dir=/etc/cni/net.d #说明： 需要添加的各配置参数（各参数的值要与系统部署的CNI插件的实际路径相对应）： --network-plugin：指定网络插件规范的类型，这里要使用CNI； --cni-bin-dir：指定CNI插件二进制程序文件的搜索目录； --cni-cache-dir：CNI插件使用的缓存目录； --cni-conf-dir：CNI插件加载配置文件的目录； 配置完成后，重载并重启cri-docker.service服务。 [root@k8s-master01 ~]# systemctl daemon-reload && systemctl restart cri-docker.service [root@k8s-master01 ~]# systemctl status cri-docker5-2、配置kubelet
#所有节点执行： #配置kubelet，为其指定cri-dockerd在本地打开的Unix Sock文件的路径，该路径一般默认为“/run/cri-dockerd.sock“ [root@k8s-master01 ~]# mkdir /etc/sysconfig [root@k8s-master01 ~]# vim /etc/sysconfig/kubelet KUBELET_KUBEADM_ARGS="--container-runtime=remote --container-runtime-endpoint=/run/cri-dockerd.sock" [root@k8s-master01 ~]# cat /etc/sysconfig/kubelet KUBELET_KUBEADM_ARGS="--container-runtime=remote --container-runtime-endpoint=/run/cri-dockerd.sock" #说明：该配置也可不进行，而是直接在后面的各kubeadm命令上使用“--cri-socket unix:///run/cri-dockerd.sock”选项6、初始化第一个主节点
#第一个主节点执行： #列出k8s所需要的镜像 [root@k8s-master01 ~]#kubeadm config images list registry.k8s.io/kube-apiserver:v1.25.4 registry.k8s.io/kube-controller-manager:v1.25.4 registry.k8s.io/kube-scheduler:v1.25.4 registry.k8s.io/kube-proxy:v1.25.4 registry.k8s.io/pause:3.8 registry.k8s.io/etcd:3.5.5-0 registry.k8s.io/coredns/coredns:v1.9.3 #使用阿里云拉取所需镜像 [root@k8s-master01 ~]#kubeadm config images pull --image-repository=registry.aliyuncs.com/google_containers --cri-socket unix:///run/cri-dockerd.sock [config/images] Pulled registry.aliyuncs.com/google_containers/kube-apiserver:v1.25.4 [config/images] Pulled registry.aliyuncs.com/google_containers/kube-controller-manager:v1.25.4 [config/images] Pulled registry.aliyuncs.com/google_containers/kube-scheduler:v1.25.4 [config/images] Pulled registry.aliyuncs.com/google_containers/kube-proxy:v1.25.4 [config/images] Pulled registry.aliyuncs.com/google_containers/pause:3.8 [config/images] Pulled registry.aliyuncs.com/google_containers/etcd:3.5.5-0 [config/images] Pulled registry.aliyuncs.com/google_containers/coredns:v1.9.3 #kubeadm可通过配置文件加载配置，以定制更丰富的部署选项。获取内置的初始配置文件的命令 kubeadm config print init-defaults [root@k8s-master01 ~]#vim kubeadm-config.yaml apiVersion: kubeadm.k8s.io/v1beta3 bootstrapTokens: kind: InitConfiguration localAPIEndpoint: # 这里的地址即为初始化的控制平面第一个节点的IP地址； advertiseAddress: 10.0.0.101 bindPort: 6443 nodeRegistration: criSocket: unix:///run/cri-dockerd.sock imagePullPolicy: IfNotPresent # 第一个控制平面节点的主机名称； name: k8s-master01.wang.org taints: - effect: NoSchedule key: node-role.kubernetes.io/master - effect: NoSchedule key: node-role.kubernetes.io/control-plane --- apiServer: timeoutForControlPlane: 4m0s apiVersion: kubeadm.k8s.io/v1beta3 # 控制平面的接入端点，我们这里选择适配到kubeapi.wang.com这一域名上； controlPlaneEndpoint: "kubeapi.wang.org:6443" certificatesDir: /etc/kubernetes/pki clusterName: kubernetes controllerManager: {} dns: {} etcd: local: dataDir: /var/lib/etcd imageRepository: registry.aliyuncs.com/google_containers kind: ClusterConfiguration kubernetesVersion: v1.25.4 networking: dnsDomain: cluster.local serviceSubnet: 10.96.0.0/12 podSubnet: 10.244.0.0/16 scheduler: {} --- apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration # 用于配置kube-proxy上为Service指定的代理模式，默认为iptables； mode: "ipvs" [root@k8s-master01 ~]#kubeadm init --config kubeadm-config.yaml --upload-certs #如提示以下信息，代表初始化完成，请记录信息，以便后续使用： ..... Your Kubernetes control-plane has initialized successfully! To start using your cluster, you need to run the following as a regular user: mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config Alternatively, if you are the root user, you can run: export KUBECONFIG=/etc/kubernetes/admin.conf You should now deploy a pod network to the cluster. Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at: https://kubernetes.io/docs/concepts/cluster-administration/addons/ You can now join any number of the control-plane node running the following command on each as root: kubeadm join kubeapi.wang.org:6443 --token c9r0oz.nfw6c83xm07hzwy6 --discovery-token-ca-cert-hash sha256:5a46d743466eac029eafae4a8204c769a7867c1e64d144f22a769c55e09da3bd --control-plane --certificate-key 900b6459a376f9000c49af401bcd12e70e55d3154aa7b71f04e891c914cf661c Please note that the certificate-key gives access to cluster sensitive data, keep it secret! As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use "kubeadm init phase upload-certs --upload-certs" to reload certs afterward. Then you can join any number of worker nodes by running the following on each as root: kubeadm join kubeapi.wang.org:6443 --token c9r0oz.nfw6c83xm07hzwy6 --discovery-token-ca-cert-hash sha256:5a46d743466eac029eafae4a8204c769a7867c1e64d144f22a769c55e09da3bd [root@k8s-master01 ~]# mkdir -p $HOME/.kube [root@k8s-master01 ~]# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config [root@k8s-master01 ~]# sudo chown $(id -u):$(id -g) $HOME/.kube/config 7、部署网络插件flanner
#所有节点执行： #下载链接： https://github.com/flannel-io/flannel/releases [root@k8s-master01 ~]# mkdir /opt/bin [root@k8s-master01 ~]# cp flanneld-amd64 /opt/bin/flanneld [root@k8s-master01 ~]# chmod +x /opt/bin/flanneld [root@k8s-master01 ~]# ll /opt/bin/flanneld -rwxr-xr-x 1 root root 39358256 11月 19 20:41 /opt/bin/flanneld*#第一个主节点执行： #部署kube-flannel [root@k8s-master01 ~]#kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml namespace/kube-flannel created clusterrole.rbac.authorization.k8s.io/flannel created clusterrolebinding.rbac.authorization.k8s.io/flannel created serviceaccount/flannel created configmap/kube-flannel-cfg created daemonset.apps/kube-flannel-ds created #确认Pod的状态为“Running” [root@k8s-master01 ~]#kubectl get pods -n kube-flannel NAME READY STATUS RESTARTS AGE kube-flannel-ds-7hqhk 1/1 Running 0 119s #此时，k8s-master01已经就绪 [root@k8s-master01 ~]# kubectl get nodes NAME STATUS ROLES AGE VERSION k8s-master01 Ready control-plane 20m v1.25.38、添加其他节点到集群中
#k8s-master02和k8s-master03执行： #k8s-master02和k8s-master03加入集群 [root@k8s-master02 ~]#kubeadm join kubeapi.wang.org:6443 --token c9r0oz.nfw6c83xm07hzwy6 --discovery-token-ca-cert-hash sha256:5a46d743466eac029eafae4a8204c769a7867c1e64d144f22a769c55e09da3bd --control-plane --certificate-key 900b6459a376f9000c49af401bcd12e70e55d3154aa7b71f04e891c914cf661c --cri-socket unix:///run/cri-dockerd.sock #注意，命令需要加上--cri-socket unix:///run/cri-dockerd.sock #注意：token要复制对，要不然会报以下错误（本人因为复制少了一个字母，折腾了一个多小时）： Downloading the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace error execution phase control-plane-prepare/download-certs: error downloading certs: error decoding certificate key: encoding/hex: odd length hex string To see the stack trace of this error execute with --v=5 or higher # 使master02、master03也能管理集群： [root@k8s-master02 ~]#mkdir -p $HOME/.kube [root@k8s-master02 ~]#sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config [root@k8s-master02 ~]#sudo chown $(id -u):$(id -g) $HOME/.kube/config [root@k8s-master02 ~]#kubectl config view apiVersion: v1 clusters: - cluster: certificate-authority-data: DATA+OMITTED server: https://kubeapi.wang.org:6443 name: kubernetes contexts: - context: cluster: kubernetes user: kubernetes-admin name: kubernetes-admin@kubernetes current-context: kubernetes-admin@kubernetes kind: Config preferences: {} users: - name: kubernetes-admin user: client-certificate-data: REDACTED client-key-data: REDACTED #此时master02和master03已经可以管理集群了，但是是通过kubeapi.wang.org这个域名连接到101管理的，为了让master02和master03连接本机管理集群，所以需要修改hosts把域名指向本机（因为没有配置单独的dns和vip） [root@k8s-master02 ~]#vim /etc/hosts 10.0.0.101 k8s-master01 k8s-master01.wang.org 10.0.0.102 k8s-master02 k8s-master02.wang.org kubeapi.wang.org kubeapi 10.0.0.103 k8s-master03 k8s-master03.wang.org#k8s-node01、k8s-node02执行 #node节点加入集群 [root@k8s-node01 ~]#kubeadm join kubeapi.wang.org:6443 --token c9r0oz.nfw6c83xm07hzwy6 --discovery-token-ca-cert-hash sha256:5a46d743466eac029eafae4a8204c769a7867c1e64d144f22a769c55e09da3bd --cri-socket unix:///run/cri-dockerd.sock #注意，命令需要加上--cri-socket unix:///run/cri-dockerd.sock#master01节点验证： [root@k8s-master01 ~]#kubectl get pods -n kube-flannel -owide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NO kube-flannel-ds-92npq 1/1 Running 0 10m 10.0.0.103 k8s-master03 <none> kube-flannel-ds-g9ch7 1/1 Running 0 5m22s 10.0.0.102 k8s-master02 <none> kube-flannel-ds-nft4j 1/1 Running 0 10m 10.0.0.101 k8s-master01.wang.org <none> kube-flannel-ds-t7z7k 1/1 Running 0 10m 10.0.0.104 k8s-node01 <none> kube-flannel-ds-z9s8w 1/1 Running 0 10m 10.0.0.105 k8s-node02 <none> 9、node节点安装nginx，实现高可用
#node01、node02执行： #数据平面使用nginx自带负载均衡： [root@k8s-node01 ~]#apt install -y nginx [root@k8s-node01 ~]#vim /etc/nginx/nginx.conf #注意stream 适合http同级别 ...... stream { upstream apiservers { server k8s-master01.wang.org:6443 max_fails=2 fail_timeout=30s; server k8s-master02.wang.org:6443 max_fails=2 fail_timeout=30s; server k8s-master03.wang.org:6443 max_fails=2 fail_timeout=30s; } server { listen 6443; proxy_pass apiservers; } } ...... [root@k8s-node01 ~]#nginx -t [root@k8s-node01 ~]#nginx -s reload [root@k8s-node01 ~]#vim /etc/hosts 10.0.0.104 k8s-node01 k8s-node01.wang.org kubeapi.wang.org kubeapi #node02解析kubeapi域名为自己 [root@k8s-node01 ~]#kubectl config view --kubeconfig=/etc/kubernetes/kubelet.conf apiVersion: v1 clusters: - cluster: certificate-authority-data: DATA+OMITTED server: https://kubeapi.wang.org:6443 name: default-cluster contexts: - context: cluster: default-cluster namespace: default user: default-auth name: default-context current-context: default-context kind: Config preferences: {} users: - name: default-auth user: client-certificate: /var/lib/kubelet/pki/kubelet-client-current.pem client-key: /var/lib/kubelet/pki/kubelet-client-current.pem10、测试
[root@k8s-master01 ~]#kubectl create deployment demoapp --image=ikubernetes/demoapp:v1.0 --replicas=2 [root@k8s-master01 ~]#kubectl get pods -owide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES demoapp-55c5f88dcb-bvfds 0/1 ContainerCreating 0 11s <none> k8s-node02 <none> <none> demoapp-55c5f88dcb-sv744 0/1 ContainerCreating 0 11s <none> k8s-node01 <none> <none> [root@k8s-master01 ~]#kubectl get pods -owide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES demoapp-55c5f88dcb-bvfds 1/1 Running 0 84s 10.244.4.2 k8s-node02 <none> <none> demoapp-55c5f88dcb-sv744 1/1 Running 0 84s 10.244.3.2 k8s-node01 <none> <none> [root@k8s-master01 ~]#kubectl create svc nodeport demoapp --tcp=80:80 [root@k8s-master01 ~]#kubectl get svc NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE demoapp NodePort 10.99.247.102 <none> 80:32606/TCP 4s kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 52m [root@k8s-master01 ~]#kubectl get ep NAME ENDPOINTS AGE demoapp 10.244.3.2:80,10.244.4.2:80 12s kubernetes 10.0.0.101:6443,10.0.0.102:6443,10.0.0.103:6443 52m [root@k8s-master01 ~]#curl 10.99.247.102 iKubernetes demoapp v1.0 !! ClientIP: 10.244.0.0, ServerName: demoapp-55c5f88dcb-bvfds, ServerIP: 10.244.4.2! [root@k8s-master01 ~]#curl 10.99.247.102 iKubernetes demoapp v1.0 !! ClientIP: 10.244.0.0, ServerName: demoapp-55c5f88dcb-sv744, ServerIP: 10.244.3.2! [root@k8s-master01 ~]#curl 10.99.247.102 iKubernetes demoapp v1.0 !! ClientIP: 10.244.0.0, ServerName: demoapp-55c5f88dcb-bvfds, ServerIP: 10.244.4.2! [root@k8s-master01 ~]#curl 10.99.247.102 iKubernetes demoapp v1.0 !! ClientIP: 10.244.0.0, ServerName: demoapp-55c5f88dcb-sv744, ServerIP: 10.244.3.2!11、命令补全
[root@k8s-master01 ~]#apt install -y bash-completion [root@k8s-master01 ~]#source /usr/share/bash-completion/bash_completion [root@k8s-master01 ~]#source <(kubectl completion bash) [root@k8s-master01 ~]#echo "source <(kubectl completion bash)" >> ~/.bashrc12、部署cs动态置备
[root@k8s-master01 ~]#apt install nfs-server nfs-common [root@k8s-master01 ~]#vim /etc/exports /data/test 10.0.0.0/24(rw,no_subtree_check,no_root_squash) [root@k8s-master01 ~]#mkdir /data/test -p [root@k8s-master01 ~]#exportfs -ar [root@k8s-master01 ~]#showmount -e 10.0.0.101 =============================================== [root@k8s-master01 ~]#kubectl create ns nfs namespace/nfs created [root@k8s-master01 ~]#kubectl get ns NAME STATUS AGE default Active 66m kube-flannel Active 59m kube-node-lease Active 66m kube-public Active 66m kube-system Active 66m nfs Active 8s [root@k8s-master01 ~]#kubectl create -f https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/deploy/example/nfs-provisioner/nfs-server.yaml -n nfs [root@k8s-master01 ~]#kubectl get pod -n nfs NAME READY STATUS RESTARTS AGE nfs-server-5847b99d99-56fhg 1/1 Running 0 28s [root@k8s-master01 ~]#curl -skSL https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/v3.1.0/deploy/install-driver.sh | bash -s v3.1.0 -- [root@k8s-master01 ~]#kubectl -n kube-system get pod -o wide -l app=csi-nfs-controller NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES csi-nfs-controller-65cf7d587-hfwk2 3/3 Running 0 2m11s 10.0.0.105 k8s-node02 <none> <none> csi-nfs-controller-65cf7d587-n6hmk 3/3 Running 0 2m11s 10.0.0.104 k8s-node01 <none> <none> [root@k8s-master01 ~]#kubectl -n kube-system get pod -o wide -l app=csi-nfs-node NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES csi-nfs-node-2mwfc 3/3 Running 0 2m23s 10.0.0.104 k8s-node01 <none> <none> csi-nfs-node-9c2j4 3/3 Running 0 2m23s 10.0.0.102 k8s-master02 <none> <none> csi-nfs-node-c4fll 3/3 Running 0 2m23s 10.0.0.103 k8s-master03 <none> <none> csi-nfs-node-k2zcv 3/3 Running 0 2m23s 10.0.0.105 k8s-node02 <none> <none> csi-nfs-node-vq2pv 3/3 Running 0 2m23s 10.0.0.101 k8s-master01.wang.org <none> <none> [root@k8s-master01 ~]#vim nfs-csi-cs.yaml apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: nfs-csi provisioner: nfs.csi.k8s.io parameters: # server: nfs-server.nfs.svc.cluster.local server: 10.0.0.101 #此处用的是真实nfs，使用官网配置pvc无法bound，一直在pending状态 share: /data/test # csi.storage.k8s.io/provisioner-secret is only needed for providing mountOptions in DeleteVolume # csi.storage.k8s.io/provisioner-secret-name: "mount-options" # csi.storage.k8s.io/provisioner-secret-namespace: "default" #reclaimPolicy: Delete reclaimPolicy: Retain volumeBindingMode: Immediate mountOptions: - hard - nfsvers=4.1 [root@k8s-master01 ~]#kubectl apply -f nfs-csi-cs.yaml [root@k8s-master01 ~]#kubectl get sc NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE nfs-csi nfs.csi.k8s.io Retain Immediate false 3s [root@k8s-master01 ~]#vim pvc-nfs-csi-dynamic.yaml --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: pvc-nfs-dynamic spec: accessModes: - ReadWriteMany resources: requests: storage: 1Gi storageClassName: nfs-csi [root@k8s-master01 ~]#kubectl apply -f pvc-nfs-csi-dynamic.yaml [root@k8s-master01 ~]#kubectl get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE pvc-nfs-dynamic Bound pvc-a693d483-8fc6-4e91-8779-6d0097b2e075 1Gi RWX nfs-csi 5s13、部署ingress
[root@k8s-master01 ~]#kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.5.1/deploy/static/provider/cloud/deploy.yaml [root@k8s-master01 ~]#kubectl get pod -n ingress-nginx NAME READY STATUS RESTARTS AGE ingress-nginx-admission-create-cv8zl 0/1 Completed 0 105s ingress-nginx-admission-patch-nmjpg 0/1 Completed 0 105s ingress-nginx-controller-8574b6d7c9-hxxj6 1/1 Running 0 105s [root@k8s-master01 ~]#kubectl get all -n ingress-nginx NAME READY STATUS RESTARTS AGE pod/ingress-nginx-admission-create-cv8zl 0/1 Completed 0 7m5s pod/ingress-nginx-admission-patch-nmjpg 0/1 Completed 0 7m5s pod/ingress-nginx-controller-8574b6d7c9-hxxj6 1/1 Running 0 7m5s NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE service/ingress-nginx-controller LoadBalancer 10.99.251.134 <pending> 80:30418/TCP,443:31389/TCP 7m6s service/ingress-nginx-controller-admission ClusterIP 10.103.59.126 <none> 443/TCP 7m6s NAME READY UP-TO-DATE AVAILABLE AGE deployment.apps/ingress-nginx-controller 1/1 1 1 7m5s NAME DESIRED CURRENT READY AGE replicaset.apps/ingress-nginx-controller-8574b6d7c9 1 1 1 7m5s NAME COMPLETIONS DURATION AGE job.batch/ingress-nginx-admission-create 1/1 16s 7m6s job.batch/ingress-nginx-admission-patch 1/1 17s 7m6s14、部署metric server
[root@k8s-master01 ~]#kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml [root@k8s-master01 ~]#kubectl get pod -n kube-system NAME READY STATUS RESTARTS AGE ...... metrics-server-8b7cc9967-ggmxc 0/1 Running 0 9s ...... #提示：如果状态一直是running但是READY状态一直未就绪，那么就执行下列操作： ===========如果状态一直是running但是READY状态一直未就绪，那么就执行下列操作=========== [root@k8s-master01 ~]#curl -LO https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml [root@k8s-master01 ~]#vim components.yaml ...... - args: - --cert-dir=/tmp - --secure-port=4443 - --kubelet-preferred-address-types=InternalIP #修改 - --kubelet-use-node-status-port - --metric-resolution=15s - --kubelet-insecure-tls #增加 image: k8s.gcr.io/metrics-server/metrics-server:v0.6.1 ...... [root@k8s-master01 ~]#kubectl apply -f components.yaml ======================================================================================== #metric就绪后就可以查询node、pod的cpu内存等信息了 [root@k8s-master01 ~]#kubectl top nodes NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% k8s-master01.wang.org 817m 40% 1113Mi 60% k8s-master02 735m 36% 1045Mi 56% k8s-master03 673m 33% 1027Mi 55% k8s-node01 314m 15% 877Mi 47% k8s-node02 260m 13% 821Mi 44% [root@k8s-master01 ~]#kubectl top pod -n ingress-nginx NAME CPU(cores) MEMORY(bytes) ingress-nginx-controller-8574b6d7c9-48pw6 2m 70Mi 15、部署dashboard
[root@k8s-master01 ~]#kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.7.0/aio/deploy/recommended.yaml [root@k8s-master01 ~]#kubectl get ns NAME STATUS AGE default Active 13h ingress-nginx Active 17m kube-flannel Active 13h kube-node-lease Active 13h kube-public Active 13h kube-system Active 13h kubernetes-dashboard Active 9s nfs Active 12h [root@k8s-master01 ~]#kubectl edit svc ingress-nginx-controller -n ingress-nginx ...... externalTrafficPolicy: Cluster externalIPs: - 10.0.0.99 ...... [root@k8s-master01 ~]#kubectl get svc -n ingress-nginx NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE ingress-nginx-controller LoadBalancer 10.99.251.134 10.0.0.99 80:30418/TCP,443:31389/TCP 20m ingress-nginx-controller-admission ClusterIP 10.103.59.126 <none> 443/TCP 20m [root@k8s-master01 ~]#vim dashboard.yaml apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: dashboard annotations: kubernetes.io/ingress.class: "nginx" ingress.kubernetes.io/ssl-passthrough: "true" nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" nginx.ingress.kubernetes.io/rewrite-target: /$2 namespace: kubernetes-dashboard spec: ingressClassName: nginx rules: - http: paths: - path: /dashboard(/|$)(.*) backend: service: name: kubernetes-dashboard port: number: 443 pathType: Prefix [root@k8s-master01 ~]#kubectl apply -f dashboard.yaml ingress.networking.k8s.io/dashboard created [root@k8s-master01 ~]#kubectl get ingress -n kubernetes-dashboard NAME CLASS HOSTS ADDRESS PORTS AGE dashboard nginx * 10.0.0.99 80 11s #创建帐户 [root@k8s-master01 ~]#vim dashboard-admin-user.yaml apiVersion: v1 kind: ServiceAccount metadata: name: admin-user namespace: kubernetes-dashboard #绑定角色为集群管理员 [root@k8s-master01 ~]#vim dashboard-admin-binding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: admin-user roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cluster-admin subjects: - kind: ServiceAccount name: admin-user namespace: kubernetes-dashboard #获取令牌 [root@k8s-master01 ~]#kubectl -n kubernetes-dashboard create token admin-user eyJhbGciOiJSUzI1NiIsImtpZCI6InBVQUp4ckNnSmxyajhLR0FKQ0ZZazZjbmZpd2hoNjY0SDRPeUhnN2JiUGMifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiXSwiZXhwIjoxNjY4OTMyMDAxLCJpYXQiOjE2Njg5Mjg0MDEsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJhZG1pbi11c2VyIiwidWlkIjoiMjhhODM4NzgtM2NmZC00YmNjLWEyMzEtMDNlZDBmNWE5YzRlIn19LCJuYmYiOjE2Njg5Mjg0MDEsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlcm5ldGVzLWRhc2hib2FyZDphZG1pbi11c2VyIn0.aZPqgYF-3-XYsATAaH6BNRJ8YpQNxoOZn5n-76UAUq_hEGzOUAqSGVkVyb8S1oK2XLMh24Ybf7bD-eB9h4JWoyptIcQmCaoeYlaoR-fglO5SfCDpfThPKGs1WI1TlZVi3mn92c4YlhweYX2mS60iQ4gdywMehj34nqoRqoqjCOf0AA8XGjUwDlcQEjgJJSOHp_XN7NP3t-4EZDCAbzfj5bvWEfhA8wxTULz-J0MDiJ9j8xBjQvx0M9GtQeXHmuCexg08o5IuBHVBoq5iAkN_t2AJJtC1VUMA9AVkSH1HbmMs4go6cGdrgM9kHvz7hpvQZ4Na5Gte52LFITHrvA3wiw
