Kafka 集群安装 (K8S)

$ vi entrypoint.sh
#!/bin/bash

NODE_ID=${HOSTNAME:6}
LISTENERS="CONTROLLER://:9093,PLAINTEXT://0.0.0.0:9092,EXTERNAL://0.0.0.0:$((30090 + $NODE_ID))"
ADVERTISED_LISTENERS="PLAINTEXT://kafka-$NODE_ID.$SERVICE.$NAMESPACE.svc.cluster.local:9092,EXTERNAL://${K8S_NODE_IP}:$((30090 + $NODE_ID))"

CONTROLLER_QUORUM_VOTERS=""
for i in $( seq 0 $REPLICAS); do
    if [[ $i != $REPLICAS ]]; then
        CONTROLLER_QUORUM_VOTERS="$CONTROLLER_QUORUM_VOTERS$i@kafka-$i.$SERVICE.$NAMESPACE.svc.cluster.local:9093,"
    else
        CONTROLLER_QUORUM_VOTERS=${CONTROLLER_QUORUM_VOTERS::-1}
    fi
done

mkdir -p $SHARE_DIR/$NODE_ID

sed -e "s+^node.id=.*+node.id=${NODE_ID}+" 
-e "s+^controller.quorum.voters=.*+controller.quorum.voters=$CONTROLLER_QUORUM_VOTERS+" 
-e "s+^listeners=.*+listeners=$LISTENERS+" 
-e "s+^advertised.listeners=.*+advertised.listeners=$ADVERTISED_LISTENERS+" 
-e "s+(^listener.security.protocol.map=.*)+1,EXTERNAL:PLAINTEXT+" 
-e "s+^log.dirs=.*+log.dirs=$SHARE_DIR/$NODE_ID+" 
/opt/kafka/config/kraft/server.properties > server.properties.updated 
&& mv server.properties.updated /opt/kafka/config/kraft/server.properties

/opt/kafka/bin/kafka-storage.sh format -t $CLUSTER_ID -c /opt/kafka/config/kraft/server.properties

exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/kraft/server.properties
#############################

$ vi Dockerfile
FROM openjdk:11

ENV KAFKA_VERSION=3.0.0
ENV SCALA_VERSION=2.13
ENV KAFKA_HOME=/opt/kafka
ENV PATH=${PATH}:${KAFKA_HOME}/bin

LABEL name="kafka" version=${KAFKA_VERSION}

RUN wget -O /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz https://downloads.apache.org/kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz 
 && tar xfz /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -C /opt 
 && rm /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz 
 && ln -s /opt/kafka_${SCALA_VERSION}-${KAFKA_VERSION} ${KAFKA_HOME} 
 && rm -rf /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz

COPY ./entrypoint.sh /
RUN ["chmod", "+x", "/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]
#############################

$ docker build -t kafka:3.0.0 .

$ mkdir ~/kafka && cd $_

$ cat > kafka-namespace.yml < 
3. 创建 PV 
# 1. 所有节点上，挂载相应的存储盘或路径
$ mkdir -p /data/kafka

# 2. 创建PV
$ cat > kafka-pv.yml < 
4. 创建 PVC 
$ cat > kafka-pvc.yml < 
5. 创建 Service(无头服务) 
$ cat > kafka-headless.yml < 
6. 创建 StatefulSet 
$ cat > kafka-app.yml < 
7. 创建 Service (外部访问) 
cat > kafka-external-svc.yml < 
8. 验证 
8.1 kafkacat 
# 安装测试工具
$ apt install kafkacat

# 获取 broker 列表
$ kafkacat -b 192.168.80.240:30090 -L
metadata for all topics (from broker -1: 192.168.80.240:30090/bootstrap):
 3 brokers:
  broker 0 at 192.168.80.242:30090
  broker 1 at 192.168.80.241:30091 (controller)
  broker 2 at 192.168.80.240:30092
 0 topics:


# 发布消息
$ kafkacat -b 192.168.80.240:30090 -t topic -P
hello world
abc
kafka test

# 订阅消息
$ kafkacat -b 192.168.80.241:30091 -t topic -C
% Reached end of topic topic [0] at offset 12
hello world
% Reached end of topic topic [0] at offset 13
abc
% Reached end of topic topic [0] at offset 14
kafka test

 
8.2 kafka 脚本 
$ kubectl exec -it kafka-0 -n kafka-cluster -- /bin/bash

> kafka-topics.sh --create --partitions 3 --replication-factor 1 --topic test --bootstrap-server kafka-0.kafka-svc.kafka-cluster.svc.cluster.local:9092,kafka-1.kafka-svc.kafka-cluster.svc.cluster.local:9092,kafka-2.kafka-svc.kafka-cluster.svc.cluster.local:9092 

> kafka-console-producer.sh --topic test --broker-list kafka-0.kafka-svc.kafka-cluster.svc.cluster.local:9092,kafka-1.kafka-svc.kafka-cluster.svc.cluster.local:9092,kafka-2.kafka-svc.kafka-cluster.svc.cluster.local:9092

> kafka-console-consumer.sh --from-beginning --topic test --bootstrap-server kafka-0.kafka-svc.kafka-cluster.svc.cluster.local:9092,kafka-1.kafka-svc.kafka-cluster.svc.cluster.local:9092,kafka-2.kafka-svc.kafka-cluster.svc.cluster.local:9092
 
9. 代码归档 
https://github.com/elihe2011/Kafka-K8S
 
参考资料：
 
https://adityasridhar.com/posts/how-to-easily-install-kafka-without-zookeeper 【主机安装kafka，不带ZK】
 
https://developer.ibm.com/tutorials/kafka-in-kubernetes/ 【kafka 安装到 k8s】
 
https://github.com/IBM/kraft-mode-kafka-on-kubernetes/blob/main/kubernetes/kafka.yml
 
https://blog.csdn.net/boling_cavalry/article/details/105466163
 
https://www.orchome.com/1903
 
https://segmentfault.com/a/1190000020715650
 
https://tsuyoshiushio.medium.com/configuring-kafka-on-kubernetes-makes-available-from-an-external-client-with-helm-96e9308ee9f4 【loadblancer】
 
https://blog.51cto.com/u_15127500/3790439 【kafkacat 使用】