1.磁盘分区（30G）
fdisk -l
fdisk /dev/sda (n->p->w)
2.在各个node上安装glusterfs客户端
yum install -y glusterfs glusterfs-fuse
在kube-apiserver的启动参数中增加--allow-priviledge=true(默认为true)
3.给各个节点打上标签
kubectl label node k8s-slave1 storagenode=gluster
kubectl label node k8s-slave2 storagenode=gluster
kubectl label node k8s-slave3 storagenode=gluster
kubectl get nodes --show-labels
4.创建glusterfs管理服务容器集群
/etc/kubernetes/manifests/kube-apiserver.yaml
--authorization-mode=RBAC //k8s默认是RBAC模式
//glusterfs-daemonset.yaml
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: glusterfs
labels:
glusterfs: daemonset
annotations:
description: GlusterFS DaemonSet
tags: glusterfs
spec:
selector:
matchLabels:
glusterfs-node: pod
template:
metadata:
name: glusterfs
labels:
glusterfs-node: pod
spec:
nodeSelector:
storagenode: glusterfs
hostNetwork: true
containers:
- image: gluster/gluster-centos:latest
imagePullPolicy: IfNotPresent
name: glusterfs
resources:
requests:
memory: 100Mi
cpu: 100m
volumeMounts:
- name: glusterfs-heketi
mountPath: "/var/lib/heketi"
- name: glusterfs-run
mountPath: "/run"
- name: glusterfs-lvm
mountPath: "/run/lvm"
- name: glusterfs-etc
mountPath: "/etc/glusterfs"
- name: glusterfs-logs
mountPath: "/var/log/glusterfs"
- name: glusterfs-config
mountPath: "/var/lib/glusterd"
- name: glusterfs-host-dev
mountPath: "/mnt/host-dev"
- name: glusterfs-misc
mountPath: "/var/lib/misc/glusterfsd"
- name: glusterfs-block-sys-class
mountPath: "/sys/class"
- name: glusterfs-block-sys-module
mountPath: "/sys/module"
- name: glusterfs-cgroup
mountPath: "/sys/fs/cgroup"
readOnly: true
- name: glusterfs-ssl
mountPath: "/etc/ssl"
readOnly: true
- name: kernel-modules
mountPath: "/lib/modules"
readOnly: true
securityContext:
capabilities: {}
privileged: true
readinessProbe:
timeoutSeconds: 3
initialDelaySeconds: 40
exec:
command:
- "/bin/bash"
- "-c"
- "if command -v /usr/local/bin/status-probe.sh; then /usr/local/bin/status-probe.sh readiness; else systemctl status glusterd.service; fi"
periodSeconds: 25
successThreshold: 1
failureThreshold: 50
livenessProbe:
timeoutSeconds: 3
initialDelaySeconds: 40
exec:
command:
- "/bin/bash"
- "-c"
- "if command -v /usr/local/bin/status-probe.sh; then /usr/local/bin/status-probe.sh liveness; else systemctl status glusterd.service; fi"
periodSeconds: 25
successThreshold: 1
failureThreshold: 50
volumes:
- name: glusterfs-heketi
hostPath:
path: "/var/lib/heketi"
- name: glusterfs-run
- name: glusterfs-lvm
hostPath:
path: "/run/lvm"
- name: glusterfs-etc
hostPath:
path: "/etc/glusterfs"
- name: glusterfs-logs
hostPath:
path: "/var/log/glusterfs"
- name: glusterfs-config
hostPath:
path: "/var/lib/glusterd"
- name: glusterfs-host-dev
hostPath:
path: "/dev"
- name: glusterfs-misc
hostPath:
path: "/var/lib/misc/glusterfsd"
- name: glusterfs-block-sys-class
hostPath:
path: "/sys/class"
- name: glusterfs-block-sys-module
hostPath:
path: "/sys/module"
- name: glusterfs-cgroup
hostPath:
path: "/sys/fs/cgroup"
- name: glusterfs-ssl
hostPath:
path: "/etc/ssl"
- name: kernel-modules
hostPath:
path: "/lib/modules"
执行 kubectl create -f glusterfs-daemonset.yaml
kubectl get po
5.所有节点加载对应模块
modprobe dm_snapshot
modprobe dm_mirror
modprobe dm_thin_pool
//查看是否加载
lsmod | grep dm_snapshot
lsmod | grep dm_mirror
lsmod | grep dm_thin_pool
6.创建Heketi服务
//创建ServiceAccount并完成RBAC授权
//heketi-service-account.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: heketi-service-account
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: heketi
rules:
- apiGroups:
- ""
resources:
- endpoints
- services
- pods
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- pods/exec
verbs:
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: heketi
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: heketi
subjects:
- kind: ServiceAccount
name: heketi-service-account
namespace: default
//部署Heketi服务
//heketi-deployment.yaml
---
kind: Service
apiVersion: v1
metadata:
name: heketi
labels:
glusterfs: heketi-service
deploy-heketi: support
annotations:
description: Exposes Heketi Service
spec:
selector:
name: deploy-heketi
ports:
- name: deploy-heketi
port: 8080
targetPort: 8080
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: heketi
labels:
glusterfs: heketi-deployment
deploy-heketi: heketi-deployment
annotations:
description: Defines how to deploy Heketi
spec:
replicas: 1
selector:
matchLabels:
name: deploy-heketi
glusterfs: heketi-pod
template:
metadata:
name: deploy-heketi
labels:
name: deploy-heketi
glusterfs: heketi-pod
spec:
serviceAccountName: heketi-service-account
containers:
- image: heketi/heketi
imagePullPolicy: IfNotPresent
name: heketi
env:
- name: HEKETI_EXECUTOR
value: kubernetes
- name: HEKETI_FSTAB
value: "/var/lib/heketi/fstab"
- name: HEKETI_SNAPSHOT_LIMIT
value: '14'
- name: HEKETI_KUBE_GLUSTER_DAEMONSET
value: "y"
- name: HEKETI_IGNORE_STALE_OPERATIONS
value: "true"
ports:
- containerPort: 8080
volumeMounts:
- name: db
mountPath: "/var/lib/heketi"
readinessProbe:
timeoutSeconds: 3
initialDelaySeconds: 3
httpGet:
path: "/hello"
port: 8080
livenessProbe:
timeoutSeconds: 3
initialDelaySeconds: 30
httpGet:
path: "/hello"
port: 8080
volumes:
- name: db
hostPath:
path: "/heketi-data"
7.docker exec -it 3a194fca3721 /bin/bash 进入容器（Ctrl+p+q可退出）
//topology.json
{
"clusters": [
{
"nodes": [
{
"node": {
"hostnames": {
"manage": [
"k8s-slave1"
],
"storage": [
"192.168.9.202"
]
},
"zone": 1
},
"devices": [
"/dev/sda3"
]
},
{
"node": {
"hostnames": {
"manage": [
"k8s-slave2"
],
"storage": [
"192.168.9.207"
]
},
"zone": 1
},
"devices": [
"/dev/sda3"
]
},
{
"node": {
"hostnames": {
"manage": [
"k8s-slave3"
],
"storage": [
"192.168.9.233"
]
},
"zone": 1
},
"devices": [
"/dev/sda3"
]
}
]
}
]
}
export HEKETI_CLI_SERVER=http://10.109.24.104:8080
heketi-cli -s $HEKETI_CLI_SERVER --user admin --secret 'My Secret' topology load --json=topolopy.json
curl http://10.109.24.104:8080/hello //集群IP访问
8.heketi-cli topology info 可看到node和device的详细信息
9.搭建mysql主从复制集群
//mysql-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: mysql
labels:
app: mysql
data:
primary.cnf: |
# Apply this config only on the primary.
[mysqld]
log-bin
replica.cnf: |
# Apply this config only on replicas.
[mysqld]
super-read-only
//mysql-storageclass.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: mysql
annotations:
storageclass.beta.kubernetes.io/is-default-class: "true"
provisioner: kubernetes.io/glusterfs
allowVolumeExpansion: true
parameters:
resturl: "http://10.109.24.104:8080"
restauthenabled: "true"
restuser: "admin"
restuserkey: "My Secret"
gidMin: "40000"
gidMax: "50000"
volumetype: "replicate:3"
//mysql-pvc.yaml
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: data-mysql-0
#annotations:
#volume.beta.kubernetes.io/storage-class: "gluster-client-provisioner"
spec:
storageClassName: mysql
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Gi
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: data-mysql-1
spec:
storageClassName: mysql
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Gi
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: data-mysql-2
spec:
storageClassName: mysql
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Gi
// mysql-services.yaml
# Headless service for stable DNS entries of StatefulSet members.
apiVersion: v1
kind: Service
metadata:
name: mysql
labels:
app: mysql
spec:
type: NodePort
ports:
- port: 3306
targetPort: 3306
selector:
app: mysql
---
# Client service for connecting to any MySQL instance for reads.
# For writes, you must instead connect to the primary: mysql-0.mysql.
apiVersion: v1
kind: Service
metadata:
name: mysql-read
labels:
app: mysql
spec:
type: NodePort
ports:
- port: 3306
targetPort: 3306
selector:
app: mysql
//mysql-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: mysql
spec:
selector:
matchLabels:
app: mysql
serviceName: mysql
replicas: 3
template:
metadata:
labels:
app: mysql
spec:
initContainers:
- name: init-mysql
image: mysql:5.7
command:
- bash
- "-c"
- |
set -ex
# Generate mysql server-id from pod ordinal index.
[[ `hostname` =~ -([0-9]+)$ ]] || exit 1
ordinal=${BASH_REMATCH[1]}
echo [mysqld] > /mnt/conf.d/server-id.cnf
# Add an offset to avoid reserved server-id=0 value.
echo server-id=$((100 + $ordinal)) >> /mnt/conf.d/server-id.cnf
# Copy appropriate conf.d files from config-map to emptyDir.
if [[ $ordinal -eq 0 ]]; then
cp /mnt/config-map/primary.cnf /mnt/conf.d/
else
cp /mnt/config-map/replica.cnf /mnt/conf.d/
fi
volumeMounts:
- name: conf
mountPath: /mnt/conf.d
- name: config-map
mountPath: /mnt/config-map
- name: clone-mysql
image: yizhiyong/xtrabackup
command:
- bash
- "-c"
- |
set -ex
# Skip the clone if data already exists.
[[ -d /var/lib/mysql/mysql ]] && exit 0
# Skip the clone on primary (ordinal index 0).
[[ `hostname` =~ -([0-9]+)$ ]] || exit 1
ordinal=${BASH_REMATCH[1]}
[[ $ordinal -eq 0 ]] && exit 0
# Clone data from previous peer.
ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql
# Prepare the backup.
xtrabackup --prepare --target-dir=/var/lib/mysql
volumeMounts:
- name: data
mountPath: /var/lib/mysql
subPath: mysql
- name: conf
mountPath: /etc/mysql/conf.d
containers:
- name: mysql
image: mysql:5.7
env:
- name: MYSQL_ALLOW_EMPTY_PASSWORD
value: "1"
ports:
- name: mysql
containerPort: 3306
volumeMounts:
- name: data
mountPath: /var/lib/mysql
subPath: mysql
- name: conf
mountPath: /etc/mysql/conf.d
resources:
requests:
cpu: 500m
memory: 1Gi
livenessProbe:
exec:
command: ["mysqladmin", "ping"]
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
readinessProbe:
exec:
# Check we can execute queries over TCP (skip-networking is off).
command: ["mysql", "-h", "127.0.0.1", "-e", "SELECt 1"]
initialDelaySeconds: 5
periodSeconds: 2
timeoutSeconds: 1
- name: xtrabackup
image: yizhiyong/xtrabackup
ports:
- name: xtrabackup
containerPort: 3307
command:
- bash
- "-c"
- |
set -ex
cd /var/lib/mysql
# Determine binlog position of cloned data, if any.
if [[ -f xtrabackup_slave_info && "x$( change_master_to.sql.in
# Ignore xtrabackup_binlog_info in this case (it's useless).
rm -f xtrabackup_slave_info xtrabackup_binlog_info
elif [[ -f xtrabackup_binlog_info ]]; then
# We're cloning directly from primary. Parse binlog position.
[[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1
rm -f xtrabackup_binlog_info xtrabackup_slave_info
echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',
MASTER_LOG_POS=${BASH_REMATCH[2]}" > change_master_to.sql.in
fi
# Check if we need to complete a clone by starting replication.
if [[ -f change_master_to.sql.in ]]; then
echo "Waiting for mysqld to be ready (accepting connections)"
until mysql -h 127.0.0.1 -e "SELECt 1"; do sleep 1; done
echo "Initializing replication from clone position"
mysql -h 127.0.0.1
-e "$(
kubectl get pv,pvc
kubectl get svc
