问题

在380节点的大集群上，运行29T数据量的HiBench测试套中ScalaSort测试用例，使用以下关键配置（--executor-cores 4）出现如下异常：

org.apache.spark.shuffle.FetchFailedException: Failed to connect to /192.168.114.12:23242    at org.apache.spark.storage.ShuffleBlockFetcherIterator.throwFetchFailedException(ShuffleBlockFetcherIterator.scala:321)    at org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:306)    at org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:51)    at scala.collection.Iterator$$anon$11.next(Iterator.scala:328)    at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371)    at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:327)    at org.apache.spark.util.CompletionIterator.hasNext(CompletionIterator.scala:32)    at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:39)    at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:217)    at org.apache.spark.shuffle.hash.HashShuffleReader.read(HashShuffleReader.scala:102)    at org.apache.spark.rdd.ShuffledRDD.compute(ShuffledRDD.scala:90)    at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:301)    at org.apache.spark.rdd.RDD.iterator(RDD.scala:265)    at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)    at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:301)    at org.apache.spark.rdd.RDD.iterator(RDD.scala:265)    at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)    at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:301)    at org.apache.spark.rdd.RDD.iterator(RDD.scala:265)    at org.apache.spark.rdd.UnionRDD.compute(UnionRDD.scala:87)    at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:301)    at org.apache.spark.rdd.RDD.iterator(RDD.scala:265)    at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)    at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)    at org.apache.spark.scheduler.Task.run(Task.scala:87)    at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)    at java.lang.Thread.run(Thread.java:745)Caused by: java.io.IOException: Failed to connect to /192.168.114.12:23242    at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:214)    at org.apache.spark.network.client.TransportClientFactory.createClient(TransportClientFactory.java:167)    at org.apache.spark.network.netty.NettyBlockTransferService$$anon$1.createAndStart(NettyBlockTransferService.scala:91)    at org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(RetryingBlockFetcher.java:140)    at org.apache.spark.network.shuffle.RetryingBlockFetcher.access$200(RetryingBlockFetcher.java:43)    at org.apache.spark.network.shuffle.RetryingBlockFetcher$1.run(RetryingBlockFetcher.java:170)    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)    at java.util.concurrent.FutureTask.run(FutureTask.java:266)    ... 3 moreCaused by: java.net.ConnectException: Connection timed out: /192.168.114.12:23242    at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)    at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)    at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:224)    at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:289)    at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:528)    at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:468)    at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:382)    at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:354)    at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)    ... 1 more