This spark exception happens randomly when I use stream.stop in spark streaming continuous mode. Spark version is 3.4.1
23/09/11 15:05:58 ERROR Utils: Aborting task
org.apache.spark.SparkException: Could not find EpochCoordinator-9280d4bb-91fe-4d1e-90b3-fa3c14a1954e--a7f3d2e5-e1cc-4c06-a358-513d083d58c2.
at org.apache.spark.rpc.netty.Dispatcher.postMessage(Dispatcher.scala:178) ~[spark-core_2.12-3.4.0.jar:3.4.0]
at org.apache.spark.rpc.netty.Dispatcher.postOneWayMessage(Dispatcher.scala:150) ~[spark-core_2.12-3.4.0.jar:3.4.0]
at org.apache.spark.rpc.netty.NettyRpcEnv.send(NettyRpcEnv.scala:193) ~[spark-core_2.12-3.4.0.jar:3.4.0]
at org.apache.spark.rpc.netty.NettyRpcEndpointRef.send(NettyRpcEnv.scala:563) ~[spark-core_2.12-3.4.0.jar:3.4.0]
at org.apache.spark.sql.execution.streaming.continuous.ContinuousQueuedDataReader.next(ContinuousQueuedDataReader.scala:119) ~[spark-sql_2.12-3.4.0.jar:3.4.0]
at org.apache.spark.sql.execution.streaming.continuous.ContinuousDataSourceRDD$$anon$1.getNext(ContinuousDataSourceRDD.scala:102) ~[spark-sql_2.12-3.4.0.jar:3.4.0]
at org.apache.spark.sql.execution.streaming.continuous.ContinuousDataSourceRDD$$anon$1.getNext(ContinuousDataSourceRDD.scala:94) ~[spark-sql_2.12-3.4.0.jar:3.4.0]
at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) ~[spark-core_2.12-3.4.0.jar:3.4.0]
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) ~[scala-library-2.12.17.jar:?]
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) ~[scala-library-2.12.17.jar:?]
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) ~[scala-library-2.12.17.jar:?]
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) ~[scala-library-2.12.17.jar:?]
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) ~[scala-library-2.12.17.jar:?]
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) ~[scala-library-2.12.17.jar:?]
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) ~[scala-library-2.12.17.jar:?]
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) ~[scala-library-2.12.17.jar:?]
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) ~[scala-library-2.12.17.jar:?]
at scala.collection.Iterator$$anon$10.hasNext(Iterator.scala:460) ~[scala-library-2.12.17.jar:?]
at org.apache.spark.sql.execution.streaming.continuous.ContinuousWriteRDD.$anonfun$compute$1(ContinuousWriteRDD.scala:60) ~[spark-sql_2.12-3.4.0.jar:3.4.0]
at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23) ~[scala-library-2.12.17.jar:?]
at org.apache.spark.util.Utils$.tryWithSafeFinallyAndFailureCallbacks(Utils.scala:1563) ~[spark-core_2.12-3.4.0.jar:3.4.0]
at org.apache.spark.sql.execution.streaming.continuous.ContinuousWriteRDD.compute(ContinuousWriteRDD.scala:91) ~[spark-sql_2.12-3.4.0.jar:3.4.0]
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:364) ~[spark-core_2.12-3.4.0.jar:3.4.0]
at org.apache.spark.rdd.RDD.iterator(RDD.scala:328) ~[spark-core_2.12-3.4.0.jar:3.4.0]
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:92) ~[spark-core_2.12-3.4.0.jar:3.4.0]
at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:161) ~[spark-core_2.12-3.4.0.jar:3.4.0]
at org.apache.spark.scheduler.Task.run(Task.scala:139) ~[spark-core_2.12-3.4.0.jar:3.4.0]
at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:554) ~[spark-core_2.12-3.4.0.jar:3.4.0]
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1529) ~[spark-core_2.12-3.4.0.jar:3.4.0]
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:557) ~[spark-core_2.12-3.4.0.jar:3.4.0]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_382]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_382]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_382]
I have tried to set config("spark.sql.streaming.stopTimeout", 0) which to shutdown stream immediately and a lot more, none of them works.