Flink on docker

https://hub.docker.com/_/flink?tab=tags&page=1&ordering=-name&name=1.13.1

docker pull flink:1.13.1-scala_2.12-java8

# 这里可以按照你的需要对镜像个性化定制,修改配置文件,增加各种依赖

FROM flink:1.13.1-scala_2.12-java8
RUN mkdir -p /opt/flink/{usrlib,flink-checkpoints,flink-savepoints} && chmod -R 777 /opt/flinkCOPY ./flink-conf.yaml /opt/flink/conf/flink-conf.yaml
RUN ln -fs /opt/flink/opt/flink-queryable-state-runtime-*.jar /opt/flink/lib/.

RUN mkdir -p /opt/flink/plugins/flink-s3-fs-hadoop
RUN ln -fs /opt/flink/opt/flink-s3-fs-hadoop-*.jar /opt/flink/plugins/flink-s3-fs-hadoop/.

docker build -t docker.flink:v1.13.1-2.12-1.8 .

FLINK_PROPERTIES="jobmanager.rpc.address: jobmanager"
# 官网docker network create flink-network
# 创建一个flink专用的网络连接,本人使用的阿里云esc

docker run 
-d  
--rm 
--name=jobmanager 
--network bridge 
--publish 8081:8081 
--env DISABLE_JEMALLOC=true 
--env FLINK_PROPERTIES="${FLINK_PROPERTIES}" 
docker.flink:v1.13.1-2.12-1.8 jobmanager

docker run 
-d  
--rm 
--name=taskmanager 
--network bridge 
--env FLINK_PROPERTIES="${FLINK_PROPERTIES}" 
--env taskmanager.numberOfTaskSlots=4 
docker.flink:v1.13.1-2.12-1.8 taskmanager

# 1. 尝试提交flink run的方式,但是flink执行脚本的本地conf/flink-conf.yaml 配置文件需要与容器的一样,否则会找不到jobmanager

# 2. web ui提交
 
# 3. 容器执行方式 麻蛋官网的JM_CONTAINER多了个括号
JOB_CLASS_NAME="xxxxxxx"
JM_CONTAINER=$(docker ps --filter name=jobmanager --format={{.ID}}))
docker cp /opt/frame/flink-1.13.1/runjob/flink-stream-service-stg-1.0-SNAPSHOT-jar-with-dependencies.jar "${JM_CONTAINER}":/job.jar
docker exec -t -i "${JM_CONTAINER}" flink run -d -c ${JOB_CLASS_NAME} /job.jar

//TODO

docker-compose
version: "2.2"
services:
  jobmanager:
    image: docker.flink:v1.13.1-2.12-1.8
    ports:
      - "8083:8083"
    command: jobmanager
    volumes:
      - /opt/flinkdata/docker-checkpoints:/opt/flink/flink-checkpoints
      - /opt/flinkdata/docker-savepoints:/opt/flink/flink-savepoints
      - /opt/flinkdata/docker-logs:/opt/flink/log
      - /opt/flinkdata/docker-runjob:/opt/flink/usrlib
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager

  taskmanager:
    image: docker.flink:v1.13.1-2.12-1.8
    depends_on:
      - jobmanager
    command: taskmanager
    volumes:
      - /opt/flinkdata/docker-checkpoints:/opt/flink/flink-checkpoints
      - /opt/flinkdata/docker-savepoints:/opt/flink/flink-savepoints
      - /opt/flinkdata/docker-logs:/opt/flink/log
      - /opt/flinkdata/docker-runjob:/opt/flink/usrlib
    scale: 1
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        taskmanager.numberOfTaskSlots: 4
networks:
  default:
    external:
      name: valheim_default

docker-compose up <-d 后台执行>

# 查看状态 
docker ps

参考上文session的记录