2022/03/18 17:36:47 HttpRequest GET http://10.1.170.115:8080/v1/statement/20220318_093646_03065_awf2q/2 map[errorClass:java.util.concurrent.TimeoutException message:Futures timed out after [600 seconds] stack:java.util.concurrent.TimeoutException: Futures timed out after [600 seconds] at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:223) at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:227) at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:190) at akka.dispatch.MonitorableThreadFactory$AkkaForkJoinWorkerThread$$anon$3.block(ThreadPoolBuilder.scala:167) at akka.dispatch.forkjoin.ForkJoinPool.managedBlock(ForkJoinPool.java:3641) at akka.dispatch.MonitorableThreadFactory$AkkaForkJoinWorkerThread.blockOn(ThreadPoolBuilder.scala:165) at akka.dispatch.BatchingExecutor$BlockableBatch.blockOn(BatchingExecutor.scala:106) at scala.concurrent.Await$.result(package.scala:190) at spark.jobserver.WebApi.spark$jobserver$WebApi$$getJobManagerForContext(WebApi.scala:812) at spark.jobserver.WebApi$$anonfun$jobRoutes$1$$anonfun$apply$94$$anonfun$apply$95.apply(WebApi.scala:723) at spark.jobserver.WebApi$$anonfun$jobRoutes$1$$anonfun$apply$94$$anonfun$apply$95.apply(WebApi.scala:713) at spray.routing.ApplyConverterInstances$$anon$17$$anonfun$apply$6.apply(ApplyConverterInstances.scala:60) at spray.routing.ApplyConverterInstances$$anon$17$$anonfun$apply$6.apply(ApplyConverterInstances.scala:59) at spray.routing.ConjunctionMagnet$$anon$3$$anon$4$$anonfun$happly$1$$anonfun$apply$1.apply(Directive.scala:38) at spray.routing.ConjunctionMagnet$$anon$3$$anon$4$$anonfun$happly$1$$anonfun$apply$1.apply(Directive.scala:37) at spray.routing.directives.BasicDirectives$$anon$2.happly(BasicDirectives.scala:79) at spray.routing.Directive$$anon$7$$anonfun$happly$4.apply(Directive.scala:86) at spray.routing.Directive$$anon$7$$anonfun$happly$4.apply(Directive.scala:86) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anonfun$mapRequestContext$1$$anonfun$apply$1.apply(BasicDirectives.scala:30) at spray.routing.directives.BasicDirectives$$anonfun$mapRequestContext$1$$anonfun$apply$1.apply(BasicDirectives.scala:30) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anonfun$mapRequestContext$1$$anonfun$apply$1.apply(BasicDirectives.scala:30) at spray.routing.directives.BasicDirectives$$anonfun$mapRequestContext$1$$anonfun$apply$1.apply(BasicDirectives.scala:30) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:32) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:31) at spray.routing.RequestContext$$anonfun$withRejectionHandling$1.applyOrElse(RequestContext.scala:130) at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:36) at spray.routing.RequestContext$$anon$1.handle(RequestContext.scala:84) at akka.spray.UnregisteredActorRefbase.$bang(UnregisteredActorRefbase.scala:72) at spray.routing.RequestContext$$anon$3.handle(RequestContext.scala:102) at akka.spray.UnregisteredActorRefbase.$bang(UnregisteredActorRefbase.scala:72) at spray.routing.RequestContext.reject(RequestContext.scala:202) at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:35) at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:34) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:32) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:31) at spray.routing.RequestContext$$anonfun$withRejectionHandling$1.applyOrElse(RequestContext.scala:130) at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:36) at spray.routing.RequestContext$$anon$1.handle(RequestContext.scala:84) at akka.spray.UnregisteredActorRefbase.$bang(UnregisteredActorRefbase.scala:72) at spray.routing.RequestContext$$anon$3.handle(RequestContext.scala:102) at akka.spray.UnregisteredActorRefbase.$bang(UnregisteredActorRefbase.scala:72) at spray.routing.RequestContext.reject(RequestContext.scala:202) at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:35) at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:34) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:32) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:31) at spray.routing.RequestContext$$anonfun$withRejectionHandling$1.applyOrElse(RequestContext.scala:130) at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:36) at spray.routing.RequestContext$$anon$1.handle(RequestContext.scala:84) at akka.spray.UnregisteredActorRefbase.$bang(UnregisteredActorRefbase.scala:72) at spray.routing.RequestContext$$anon$3.handle(RequestContext.scala:102) at akka.spray.UnregisteredActorRefbase.$bang(UnregisteredActorRefbase.scala:72) at spray.routing.RequestContext.reject(RequestContext.scala:202) at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:35) at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:34) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:32) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:31) at spray.routing.RequestContext$$anonfun$withRejectionHandling$1.applyOrElse(RequestContext.scala:130) at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:36) at spray.routing.RequestContext$$anon$1.handle(RequestContext.scala:84) at akka.spray.UnregisteredActorRefbase.$bang(UnregisteredActorRefbase.scala:72) at spray.routing.RequestContext$$anon$3.handle(RequestContext.scala:102) at akka.spray.UnregisteredActorRefbase.$bang(UnregisteredActorRefbase.scala:72) at spray.routing.RequestContext.reject(RequestContext.scala:202) at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:35) at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:34) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:32) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:31) at spray.routing.RequestContext$$anonfun$withRejectionHandling$1.applyOrElse(RequestContext.scala:130) at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:36) at spray.routing.RequestContext$$anon$1.handle(RequestContext.scala:84) at akka.spray.UnregisteredActorRefbase.$bang(UnregisteredActorRefbase.scala:72) at spray.routing.RequestContext$$anon$3.handle(RequestContext.scala:102) at akka.spray.UnregisteredActorRefbase.$bang(UnregisteredActorRefbase.scala:72) at spray.routing.RequestContext.reject(RequestContext.scala:202) at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:35) at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:34) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:32) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:31) at spray.routing.RequestContext$$anonfun$withRejectionHandling$1.applyOrElse(RequestContext.scala:130) at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:36) at spray.routing.RequestContext$$anon$1.handle(RequestContext.scala:84) at akka.spray.UnregisteredActorRefbase.$bang(UnregisteredActorRefbase.scala:72) at spray.routing.RequestContext$$anon$3.handle(RequestContext.scala:102) at akka.spray.UnregisteredActorRefbase.$bang(UnregisteredActorRefbase.scala:72) at spray.routing.RequestContext.reject(RequestContext.scala:202) at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:35) at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:34) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:32) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1$$anonfun$apply$1.apply(RouteConcatenation.scala:31) at spray.routing.RequestContext$$anonfun$withRejectionHandling$1.applyOrElse(RequestContext.scala:130) at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:36) at spray.routing.RequestContext$$anon$1.handle(RequestContext.scala:84) at akka.spray.UnregisteredActorRefbase.$bang(UnregisteredActorRefbase.scala:72) at spray.routing.RequestContext.reject(RequestContext.scala:202) at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:35) at spray.routing.directives.RouteDirectives$$anon$1.apply(RouteDirectives.scala:34) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.directives.BasicDirectives$$anon$3$$anonfun$happly$1.apply(BasicDirectives.scala:92) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:30) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:29) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:30) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:29) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:30) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:29) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:30) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:29) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:30) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:29) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:30) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:29) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:30) at spray.routing.RouteConcatenation$RouteConcatenation$$anonfun$$tilde$1.apply(RouteConcatenation.scala:29) at spray.routing.directives.OnSuccessFutureMagnet$$anon$2$$anonfun$happly$2$$anonfun$apply$2.apply(FutureDirectives.scala:81) at spray.routing.directives.OnSuccessFutureMagnet$$anon$2$$anonfun$happly$2$$anonfun$apply$2.apply(FutureDirectives.scala:79) at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:36) at akka.dispatch.BatchingExecutor$AbstractBatch.processBatch(BatchingExecutor.scala:55) at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply$mcV$sp(BatchingExecutor.scala:91) at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply(BatchingExecutor.scala:91) at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply(BatchingExecutor.scala:91) at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72) at akka.dispatch.BatchingExecutor$BlockableBatch.run(BatchingExecutor.scala:90) at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:40) at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:43) at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) ] 2022/03/18 17:37:33 [-] Plugin:ParseInputTablePlugin @ 0.0.1 ------------------------- 2022/03/18 17:37:33 [-] Plugin:ContextSavePlugin @ 0.0.2 ------------------------- 2022/03/18 17:37:33 [-] Plugin:AuditLogPlugin @ 0.0.1 -------------------------定位
可能性1，初步判断Spark在获取结果数据时，数据量过大导致超时可能性2，当Spark 进行broadcast join时，其中一个表数据过大，导致数据拉取时超过broadcast join的广播时间 解决
调整参数
spark.sql.broadcastTimeout=3600
参考来源
https://stackoverflow.com/questions/41123846/why-does-join-fail-with-java-util-concurrent-timeoutexception-futures-timed-ou
