19/03/31 17:37:15 WARN TaskSetManager: Lost task 672.0 in stage 12.0 (TID 12473, hadoop2709.jd.163.org, executor 10): FetchFailed(BlockManagerId(14, hadoop2709.jd.163.org, 7337, None), shuffleId=8, mapId=167, reduceId=672, message=


org.apache.spark.shuffle.FetchFailedException: failed to allocate 16777216 byte(s) of direct memory (used: 1073741824, max: 1073741824)


    at org.apache.spark.storage.ShuffleBlockFetcherIterator.throwFetchFailedException(ShuffleBlockFetcherIterator.scala:528)


    at org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:459)


    at org.apache.spark.storage.ShuffleBlockFetcherIterator.next(ShuffleBlockFetcherIterator.scala:62)


    at scala.collection.Iterator$$anon$12.nextCur(Iterator.scala:434)


    at scala.collection.Iterator$$anon$12.hasNext(Iterator.scala:440)


    at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)


    at org.apache.spark.util.CompletionIterator.hasNext(CompletionIterator.scala:30)


    at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37)


    at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:408)


    at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage17.sort_addToSorter_0$(Unknown Source)


    at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage17.processNext(Unknown Source)


    at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)


    at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$10$$anon$1.hasNext(WholeStageCodegenExec.scala:614)


    at org.apache.spark.sql.execution.RowIteratorFromScala.advanceNext(RowIterator.scala:83)


    at org.apache.spark.sql.execution.joins.SortMergeJoinScanner.advancedBufferedToRowWithNullFreeJoinKey(SortMergeJoinExec.scala:810)


    at org.apache.spark.sql.execution.joins.SortMergeJoinScanner.<init>(SortMergeJoinExec.scala:685)


    at org.apache.spark.sql.execution.joins.SortMergeJoinExec$$anonfun$doExecute$1.apply(SortMergeJoinExec.scala:213)


    at org.apache.spark.sql.execution.joins.SortMergeJoinExec$$anonfun$doExecute$1.apply(SortMergeJoinExec.scala:150)


    at org.apache.spark.rdd.ZippedPartitionsRDD2.compute(ZippedPartitionsRDD.scala:89)


    at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)


    at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)


    at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:49)


    at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)


    at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)


    at org.apache.spark.rdd.ZippedPartitionsRDD2.compute(ZippedPartitionsRDD.scala:89)


    at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)


    at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)


    at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:49)


    at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)


    at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)


    at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:49)


    at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)


    at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)


    at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)


    at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)


    at org.apache.spark.scheduler.Task.run(Task.scala:109)


    at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)


    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)


    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)


    at java.lang.Thread.run(Thread.java:745)