K8s资源调度（nodeSelector&nodeAffinity&Taint&Tolerations）

• nodeSelector
• nodeAffinity
• Taint与Tolerations

Pod.spec.nodeSelector是通过kubernetes的label-selector机制进行节点选择，由scheduler调度策略MatchNodeSelector进行label匹配，调度pod到目标节点，该匹配规则是强制约束。
示例：

//给node2打上app=nginx的标签
[root@master ~]# kubectl label nodes node2.example.com  app=nginx
node/node2.example.com labeled
[root@master ~]# kubectl get  nodes node2.example.com  --show-labels
NAME                STATUS   ROLES    AGE   VERSION   LABELS
node2.example.com   Ready       24h   v1.20.0   app=nginx,beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=node2.example.com,kubernetes.io/os=linux


//创建pod
[root@master ~]# cat test.yml 
apiVersion: v1
kind: Pod
metadata: 
  name: nginx
  namespace: default
spec: 
  containers:
  - name: nginx
    image: nginx
    imagePullPolicy: IfNotPresent
  nodeSelector:
    app: nginx

[root@master ~]# kubectl apply -f test.yml 
pod/nginx created

//查看，pod在node2节点上运行
[root@master ~]# kubectl get pod nginx -o wide
NAME    READY   STATUS              RESTARTS   AGE   IP       NODE                NOMINATED NODE   READINESS GATES
nginx   0/1     ContainerCreating   0          34s      node2.example.com              

示例2：

//删除node2节点的标签
[root@master ~]# kubectl label nodes node2.example.com  app-
node/node2.example.com labeled

[root@master ~]# kubectl get nodes node2.example.com --show-labels
NAME                STATUS   ROLES    AGE   VERSION   LABELS
node2.example.com   Ready       24h   v1.20.0   beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=node2.example.com,kubernetes.io/os=linux

//创建pod
[root@master ~]# kubectl apply -f test.yml 
pod/nginx created

//查看发现处于pending（等待）状态，没有在任何节点运行
[root@master ~]# kubectl get pod nginx -o wide
NAME    READY   STATUS    RESTARTS   AGE   IP       NODE     NOMINATED NODE   READINESS GATES
nginx   0/1     Pending   0          19s                    


//给node1打上app=nginx的标签
[root@master ~]# kubectl label nodes node1.example.com  app=nginx
node/node1.example.com labeled

//再次查看pod，在node1上面运行
[root@master ~]# kubectl get pod nginx -o wide
NAME    READY   STATUS              RESTARTS   AGE     IP       NODE                NOMINATED NODE   READINESS GATES
nginx   0/1     ContainerCreating   0          2m22s      node1.example.com              

NodeAffinity意为Node亲和性调度策略。是用于替换NodeSelector的全新调度策略。目前有两种节点节点亲和性表达：

RequiredDuringSchedulingIgnoredDuringExecution:
必须满足制定的规则才可以调度pode到Node上。相当于硬限制
PreferredDuringSchedulingIgnoreDuringExecution:
强调优先满足制定规则，调度器会尝试调度pod到Node上，但并不强求，相当于软限制。多个优先级规则还可以设置权重值，以定义执行的先后顺序。

IgnoredDuringExecution的意思是：
如果一个pod所在的节点在pod运行期间标签发生了变更，不在符合该pod的节点亲和性需求，则系统将忽略Node上Lable的变化，该pod能机选在该节点运行。
示例1：

//node1上面打两个标签
[root@master ~]# kubectl label nodes node1.example.com   gpu=nvdia
node/node1.example.com labeled

//node2上面打一个标签
[root@master ~]# kubectl label nodes node2.example.com   app=nginx
node/node2.example.com labeled


[root@master ~]# kubectl get nodes node1.example.com node2.example.com --show-labels
NAME                STATUS   ROLES    AGE   VERSION   LABELS
node1.example.com   Ready       24h   v1.20.0   app=nginx,beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,gpu=nvdia,kubernetes.io/arch=amd64,kubernetes.io/hostname=node1.example.com,kubernetes.io/os=linux
node2.example.com   Ready       24h   v1.20.0   app=nginx,beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=node2.example.com,kubernetes.io/os=linux

[root@master ~]# cat test.yml 
apiVersion: v1
kind: Pod
metadata: 
  name: nginx
  namespace: default
spec: 
  containers:
  - name: nginx
    image: nginx
    imagePullPolicy: IfNotPresent
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
        - matchexpressions:
          - key: app
            operator: In
            values:
            - nginx
      preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 3
        preference:
          matchexpressions:
          - key: gpu
            operator: In
            values:
            - nvdia

[root@master ~]# kubectl apply -f test.yml 
pod/nginx created

//查看，在node1上面运行
[root@master ~]# kubectl get pod nginx -o wide
NAME    READY   STATUS              RESTARTS   AGE   IP       NODE                NOMINATED NODE   READINESS GATES
nginx   0/1     ContainerCreating   0          17s      node1.example.com              
[root@master ~]# 

示例2：

//删除必须满足
[root@master ~]# cat test.yml 
apiVersion: v1
kind: Pod
metadata: 
  name: nginx
  namespace: default
spec: 
  containers:
  - name: nginx
    image: nginx
    imagePullPolicy: IfNotPresent
  affinity:
    nodeAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
      - weight: 3
        preference:
          matchexpressions:
          - key: gpu
            operator: In
            values:
            - nvdia

[root@master ~]# kubectl apply -f test.yml 
pod/nginx created

[root@master ~]# kubectl get pod nginx -o wide
NAME    READY   STATUS              RESTARTS   AGE    IP       NODE                NOMINATED NODE   READINESS GATES
nginx   0/1     ContainerCreating   0          2m5s      node1.example.com              

使用kubectl taint命令可以给某个Node节点设置污点，Node被设置上污点之后就和Pod之间存在了一种相斥的关系，可以让Node拒绝Pod的调度执行，甚至将Node已经存在的Pod驱逐出去

污点的组成：key=value:effect
每个污点有一个 key 和 value 作为污点的标签，其中 value 可以为空，effect 描述污点的作用。

effect支持如下三个选项：
NoSchedule ：表示k8s将不会将Pod调度到具有该污点的Node上
PreferNoSchedule ：表示k8s将尽量避免将Pod调度到具有该污点的Node上
NoExecute ：表示k8s将不会将Pod调度到具有该污点的Node上，同时会将Node上已经存在的Pod驱逐出去

示例1：

//给node1加上污点
[root@master ~]# kubectl taint node node1.example.com node1:NoSchedule
node/node1.example.com tainted
[root@master ~]# kubectl describe  node node1.example.com | grep -i taint
Taints:             node1:NoSchedule


[root@master ~]# cat test.yml 
apiVersion: v1
kind: Pod
metadata: 
  name: nginx
  namespace: default
spec: 
  containers:
  - name: nginx
    image: nginx
    imagePullPolicy: IfNotPresent


[root@master ~]# kubectl apply -f test.yml 
pod/nginx created


[root@master ~]# kubectl get pod nginx -o wide
NAME    READY   STATUS              RESTARTS   AGE   IP       NODE                NOMINATED NODE   READINESS GATES
nginx   0/1     ContainerCreating   0          22s      node2.example.com              

//删除污点
[root@master ~]# kubectl taint node node1.example.com node1:NoSchedule-
node/node1.example.com untainted

示例2：

//给node2打上NoExecute的污点
[root@master ~]# kubectl taint node node2.example.com node2:NoExecute
node/node2.example.com tainted

//正在驱逐已有的pod
[root@master ~]# kubectl get pod nginx -o wide
NAME    READY   STATUS        RESTARTS   AGE    IP       NODE                NOMINATED NODE   READINESS GATES
nginx   0/1     Terminating   0          4m6s      node2.example.com              

示例3：

//给pod配上容忍
[root@master ~]# cat test.yml 
apiVersion: apps/v1
kind: Deployment
metadata: 
  name: nginx
  namespace: default
spec:
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx 
    spec:
      containers:
      - name: web1
        image: busybox
        imagePullPolicy: IfNotPresent
        command: ["bin/sh","-c","sleep 600"]
      tolerations:
      - key: node2
        effect: NoExecute

//node2上面打上污点
[root@master ~]# kubectl describe  node node2.example.com | grep -i taint
Taints:             node2:NoExecute

[root@master ~]# kubectl apply -f test.yml 
deployment.apps/nginx created

//可以在node2节点运行
[root@master ~]# kubectl get pod  -o wide
NAME                     READY   STATUS              RESTARTS   AGE   IP       NODE                NOMINATED NODE   READINESS GATES
nginx-68f57b689f-wmklh   0/1     ContainerCreating   0          29s      node2.example.com