flink 1.14.4
方法Configuration configuration = new Configuration(); configuration.setString(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, "xxx"); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(configuration);源码分析
org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
public JobExecutionResult execute(StreamGraph streamGraph) throws Exception {
JobClient jobClient = this.executeAsync(streamGraph);
...
}
@Internal
public JobClient executeAsync(StreamGraph streamGraph) throws Exception {
PipelineExecutorFactory executorFactory = this.executorServiceLoader.getExecutorFactory(this.configuration);
CompletableFuture jobClientFuture = executorFactory.getExecutor(this.configuration).execute(streamGraph, this.configuration, this.userClassloader);
try {
JobClient jobClient = (JobClient)jobClientFuture.get();
...
}
org.apache.flink.client.program.PerJobMiniClusterFactory
public CompletableFuturesubmitJob(JobGraph jobGraph, ClassLoader userCodeClassloader) throws Exception { MiniClusterConfiguration miniClusterConfig = this.getMiniClusterConfig(jobGraph.getMaximumParallelism()); MiniCluster miniCluster = (MiniCluster)this.miniClusterFactory.apply(miniClusterConfig); miniCluster.start(); return miniCluster.submitJob(jobGraph).thenApplyAsync(FunctionUtils.uncheckedFunction((submissionResult) -> { ClientUtils.waitUntilJobInitializationFinished(() -> { return (JobStatus)miniCluster.getJobStatus(submissionResult.getJobID()).get(); }, () -> { return (JobResult)miniCluster.requestJobResult(submissionResult.getJobID()).get(); }, userCodeClassloader); return submissionResult; })).thenApply((result) -> { return new MiniClusterJobClient(result.getJobID(), miniCluster, userCodeClassloader, JobFinalizationBehavior.SHUTDOWN_CLUSTER); }).whenComplete((ignored, throwable) -> { if (throwable != null) { shutDownCluster(miniCluster); } }).thenApply(Function.identity()); }
org.apache.flink.client.deployment.executors.LocalExecutor
public CompletableFutureexecute(Pipeline pipeline, Configuration configuration, ClassLoader userCodeClassloader) throws Exception { Configuration effectiveConfig = new Configuration(); effectiveConfig.addAll(this.configuration); effectiveConfig.addAll(configuration); JobGraph jobGraph = this.getJobGraph(pipeline, effectiveConfig); return PerJobMiniClusterFactory.createWithFactory(effectiveConfig, this.miniClusterFactory).submitJob(jobGraph, userCodeClassloader); } private JobGraph getJobGraph(Pipeline pipeline, Configuration configuration) throws MalformedURLException { if (pipeline instanceof Plan) { Plan plan = (Plan)pipeline; int slotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, plan.getMaximumParallelism()); int numTaskManagers = configuration.getInteger("local.number-taskmanager", 1); plan.setDefaultParallelism(slotsPerTaskManager * numTaskManagers); } return PipelineExecutorUtils.getJobGraph(pipeline, configuration); }
- 创建JobGraph,检查配置中是否存在配置项**$internal.pipeline.job-id**,如果存在则作为jobId设置
org.apache.flink.client.deployment.executors.PipelineExecutorUtils
public static JobGraph getJobGraph(@Nonnull Pipeline pipeline, @Nonnull Configuration configuration) throws MalformedURLException {
Preconditions.checkNotNull(pipeline);
Preconditions.checkNotNull(configuration);
ExecutionConfigAccessor executionConfigAccessor = ExecutionConfigAccessor.fromConfiguration(configuration);
JobGraph jobGraph = FlinkPipelineTranslationUtil.getJobGraph(pipeline, configuration, executionConfigAccessor.getParallelism());
configuration.getOptional(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID).ifPresent((strJobID) -> {
jobGraph.setJobID(JobID.fromHexString(strJobID));
});
jobGraph.addJars(executionConfigAccessor.getJars());
jobGraph.setClasspaths(executionConfigAccessor.getClasspaths());
jobGraph.setSavepointRestoreSettings(executionConfigAccessor.getSavepointRestoreSettings());
return jobGraph;
}



