Solr 7.4 org.apache.solr.common.SolrException Error opening new searcher - solr
I have setup Solr 7.4 cluster with 2 nodes.
Sometimes, when one of the nodes resets or crashes, I encounter this error and I do not know why this happens!!!??
I have to add a new replica for this Shard and remove the broken Replica to solve this problem
update :
I found this log :
ERROR (qtp1543148593-14719) [ ] o.a.s.s.HttpSolrCall null:org.apache.solr.core.SolrCoreInitializationException: SolrCore 'facebook_prime_archive_shard1_replica_n1' is not available due to init failure: Error opening new searcher
at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1590)
at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:249)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:469)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:377)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:323)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1634)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:219)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:126)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:335)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at org.eclipse.jetty.server.Server.handle(Server.java:531)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:352)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:281)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:760)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:678)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.common.SolrException: Error opening new searcher
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1012)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:867)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1135)
at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:681)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
Caused by: org.apache.solr.common.SolrException: Error opening new searcher
at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2126)
at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:2246)
at org.apache.solr.core.SolrCore.initSearcher(SolrCore.java:1095)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:984)
... 9 more
Caused by: org.apache.lucene.index.IndexNotFoundException: no segments* file found in LockValidatingDirectoryWrapper(NRTCachingDirectory(MMapDirectory#/home/solr/data/facebook_prime_archive_shard1_replica_n1/data/index.20190106105338751 lockFactory=org.apache.lucene.store.NativeFSLockFactory#136d9ea2; maxCacheMB=48.0 maxMergeSizeMB=4.0)): files: [_1rpu.dii, _1rpu.dim, _1rpu.fdt, _1rpu.fdx, _1rpu.fnm, _1rpu.nvd, _1rpu.nvm, _1rpu.si, _1rpu_7xm.liv, _1rpu_Lucene50_0.doc, _1rpu_Lucene50_0.pos, _1rpu_Lucene50_0.tim, _1rpu_Lucene50_0.tip, _1rpu_Lucene70_0.dvd, _1rpu_Lucene70_0.dvm, _2mq1.dii, _2mq1.dim, _2mq1.fdt, _2mq1.fdx, _2mq1.fnm, _2mq1.nvd, _2mq1.nvm, _2mq1.si, _2mq1_8i5.liv, _2mq1_Lucene50_0.doc, _2mq1_Lucene50_0.pos, _2mq1_Lucene50_0.tim, _2mq1_Lucene50_0.tip, _2mq1_Lucene70_0.dvd, _2mq1_Lucene70_0.dvm, _3ljq.dii, _3ljq.dim, _3ljq.fdt, _3ljq.fdx, _3ljq.fnm, _3ljq.nvd, _3ljq.nvm, _3ljq.si, _3ljq_8iz.liv, _3ljq_Lucene50_0.doc, _3ljq_Lucene50_0.pos, _3ljq_Lucene50_0.tim, _3ljq_Lucene50_0.tip, _3ljq_Lucene70_0.dvd, _3ljq_Lucene70_0.dvm, _496c.dii, _496c.dim, _496c.fdt, _496c.fdx, _496c.fnm, _496c.nvd, _496c.nvm, _496c.si, _496c_Lucene50_0.doc, _496c_Lucene50_0.pos, _496c_Lucene50_0.tim, _496c_Lucene50_0.tip, _496c_Lucene70_0.dvd, _496c_Lucene70_0.dvm, _496c_d9u.liv, _4xpr.dii, _4xpr.dim, _4xpr.fdt, _4xpr.fdx, _4xpr.fnm, _4xpr.nvd, _4xpr.nvm, _4xpr.si, _4xpr_Lucene50_0.doc, _4xpr_Lucene50_0.pos, _4xpr_Lucene50_0.tim, _4xpr_Lucene50_0.tip, _4xpr_Lucene70_0.dvd, _4xpr_Lucene70_0.dvm, _4xpr_geo.liv, _6cxz.dii, _6cxz.dim, _6cxz.fdt, _6cxz.fdx, _6cxz.fnm, _6cxz.nvd, _6cxz.nvm, _6cxz.si, _6cxz_Lucene50_0.doc, _6cxz_Lucene50_0.pos, _6cxz_Lucene50_0.tim, _6cxz_Lucene50_0.tip, _6cxz_Lucene70_0.dvd, _6cxz_Lucene70_0.dvm, _6cxz_f2a.liv, _7psg.dim, _7psg.fdt, _7psg.fdx, _7psg.nvd, _7psg.si, _7psg_Lucene50_0.doc, _7psg_Lucene50_0.pos, _7psg_Lucene50_0.tim, _7psg_Lucene50_0.tip, _7psg_Lucene70_0.dvd, _7psg_Lucene70_0.dvm, write.lock]
at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:832)
at org.apache.solr.update.SolrIndexWriter.<init>(SolrIndexWriter.java:124)
at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:97)
at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:257)
at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:131)
at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:2087)
... 12 more
Related
Flink StreamingFileSink API Exception
In the Below code snippet the stream fails with exception as shown below. But when the dependencies passed (outputPath1: String, configurationParameters1: ParameterTool, timeConversion1: TimeConversion) to the class are used and the created dependencies inside class are commented it does not fail. Unable to figure out exact reason by looking at the logs because exception in logs is not making sense with the above observation. Below BucketAssigner will be used in StreamingFileSink Connector class S3BucketAssigner(outputPath1: String, configurationParameters1: ParameterTool, timeConversion1: TimeConversion) extends BucketAssigner[String, String]{ #transient lazy val timeConversion = new TimeConversion() #transient lazy val configurations: ConfigurationProcessor = new ConfigurationProcessor( Option("testConfig.properties"), Option("testRules.properties") ) //Stream-Core Lib #transient lazy val configurationParameters: ParameterTool = configurations.getConfigurationParameters #transient lazy val outputPath = configurationParameters.getRequired("s3.output.path") override def getBucketId(in: String, context: Context): String = { val bucketIdPrefix = configurationParameters.getRequired("s3.bucket.id.prefix") val currentProcessingTimeUTC = System.currentTimeMillis() //Instant.now.toEpochMilli try { log.debug( " :: StreamingFileSink: " + outputPath ) bucketIdPrefix + timeConversion.convertTimestampToRunDate_HHMM(currentProcessingTimeUTC) } catch { case exception: Exception => log.warn( " :: StreamingFileSink: " + exception.getMessage + " outputPath: " + outputPath ) bucketIdPrefix + currentProcessingTimeUTC } } override def getSerializer: SimpleVersionedSerializer[String] = SimpleVersionedStringSerializer.INSTANCE } Some times stream fails with the 1st type of exception or sometimes with second type One possibility may be that we could think of is taskmanager running out of memory because of the way i have created the dependencies object inside class and hence it is lost. 1 Flink Stream is failing with the below exception Error from Jobmanager org.apache.flink.client.program.ProgramInvocationException: The main method caused an error: org.apache.flink.client.program.ProgramInvocationException: Job failed (JobID: b8f84bbb6c51b6f4b8773c677845a9a8) at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:372) at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:222) at org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:114) at org.apache.flink.client.cli.CliFrontend.executeProgram(CliFrontend.java:812) at org.apache.flink.client.cli.CliFrontend.run(CliFrontend.java:246) at org.apache.flink.client.cli.CliFrontend.parseAndRun(CliFrontend.java:1054) at org.apache.flink.client.cli.CliFrontend.lambda$main$10(CliFrontend.java:1132) at org.apache.flink.runtime.security.contexts.NoOpSecurityContext.runSecured(NoOpSecurityContext.java:28) at org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:1132) Caused by: java.util.concurrent.ExecutionException: org.apache.flink.client.program.ProgramInvocationException: Job failed (JobID: b8f84bbb6c51b6f4b8773c677845a9a8) at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:357) at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1908) at org.apache.flink.client.program.StreamContextEnvironment.getJobExecutionResult(StreamContextEnvironment.java:123) at org.apache.flink.client.program.StreamContextEnvironment.execute(StreamContextEnvironment.java:80) at org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.execute(StreamExecutionEnvironment.java:1916) at org.apache.flink.streaming.api.scala.StreamExecutionEnvironment.execute(StreamExecutionEnvironment.scala:839) at net.driver.StreamingJob$.main(StreamingJob.scala:53) at net.driver.StreamingJob.main(StreamingJob.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:355) ... 8 more Caused by: org.apache.flink.client.program.ProgramInvocationException: Job failed (JobID: b8f84bbb6c51b6f4b8773c677845a9a8) at org.apache.flink.client.deployment.ClusterClientJobClientAdapter.lambda$null$6(ClusterClientJobClientAdapter.java:125) at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:616) at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:591) at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1975) at org.apache.flink.util.concurrent.FutureUtils.lambda$retryOperationWithDelay$9(FutureUtils.java:403) at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774) at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750) at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1975) at org.apache.flink.client.program.rest.RestClusterClient.lambda$pollResourceAsync$26(RestClusterClient.java:698) at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774) at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750) at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1975) at org.apache.flink.util.concurrent.FutureUtils.lambda$retryOperationWithDelay$9(FutureUtils.java:403) at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774) at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750) at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) at java.util.concurrent.CompletableFuture.postFire(CompletableFuture.java:575) at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:943) at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:456) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:750) Caused by: org.apache.flink.runtime.client.JobExecutionException: Job execution failed. at org.apache.flink.runtime.jobmaster.JobResult.toJobExecutionResult(JobResult.java:144) at org.apache.flink.client.deployment.ClusterClientJobClientAdapter.lambda$null$6(ClusterClientJobClientAdapter.java:123) ... 24 more Caused by: org.apache.flink.runtime.JobException: Recovery is suppressed by FailureRateRestartBackoffTimeStrategy(FailureRateRestartBackoffTimeStrategy(failuresIntervalMS=120000,backoffTimeMS=20000,maxFailuresPerInterval=3) at org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.handleFailure(ExecutionFailureHandler.java:138) at org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.getFailureHandlingResult(ExecutionFailureHandler.java:82) at org.apache.flink.runtime.scheduler.DefaultScheduler.handleTaskFailure(DefaultScheduler.java:252) at org.apache.flink.runtime.scheduler.DefaultScheduler.maybeHandleTaskFailure(DefaultScheduler.java:242) at org.apache.flink.runtime.scheduler.DefaultScheduler.updateTaskExecutionStateInternal(DefaultScheduler.java:233) at org.apache.flink.runtime.scheduler.SchedulerBase.updateTaskExecutionState(SchedulerBase.java:684) at org.apache.flink.runtime.scheduler.UpdateSchedulerNgOnInternalFailuresListener.notifyTaskFailure(UpdateSchedulerNgOnInternalFailuresListener.java:51) at org.apache.flink.runtime.executiongraph.DefaultExecutionGraph.notifySchedulerNgAboutInternalTaskFailure(DefaultExecutionGraph.java:1473) at org.apache.flink.runtime.executiongraph.Execution.processFail(Execution.java:1133) at org.apache.flink.runtime.executiongraph.Execution.processFail(Execution.java:1073) at org.apache.flink.runtime.executiongraph.Execution.markFailed(Execution.java:912) at org.apache.flink.runtime.executiongraph.ExecutionVertex.markFailed(ExecutionVertex.java:474) at org.apache.flink.runtime.scheduler.DefaultExecutionVertexOperations.markFailed(DefaultExecutionVertexOperations.java:41) at org.apache.flink.runtime.scheduler.DefaultScheduler.handleTaskDeploymentFailure(DefaultScheduler.java:609) at org.apache.flink.runtime.scheduler.DefaultScheduler.lambda$assignAllResourcesAndRegisterProducedPartitions$6(DefaultScheduler.java:481) at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:836) at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:811) at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990) at org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolBridge$PendingRequest.failRequest(DeclarativeSlotPoolBridge.java:545) at org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolBridge.cancelPendingRequests(DeclarativeSlotPoolBridge.java:127) at org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolBridge.failPendingRequests(DeclarativeSlotPoolBridge.java:355) at org.apache.flink.runtime.jobmaster.slotpool.DeclarativeSlotPoolBridge.notifyNotEnoughResourcesAvailable(DeclarativeSlotPoolBridge.java:344) at org.apache.flink.runtime.jobmaster.JobMaster.notifyNotEnoughResourcesAvailable(JobMaster.java:809) at sun.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRpcInvocation$0(AkkaRpcActor.java:308) at org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:83) at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcInvocation(AkkaRpcActor.java:307) at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:217) at org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78) at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:163) at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24) at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20) at scala.PartialFunction.applyOrElse(PartialFunction.scala:123) at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122) at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20) at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171) at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172) at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172) at akka.actor.Actor.aroundReceive(Actor.scala:537) at akka.actor.Actor.aroundReceive$(Actor.scala:535) at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220) at akka.actor.ActorCell.receiveMessage(ActorCell.scala:580) at akka.actor.ActorCell.invoke(ActorCell.scala:548) at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270) at akka.dispatch.Mailbox.run(Mailbox.scala:231) at akka.dispatch.Mailbox.exec(Mailbox.scala:243) at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175) Caused by: java.util.concurrent.CompletionException: java.util.concurrent.CompletionException: org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException: Could not acquire the minimum required resources. at org.apache.flink.runtime.scheduler.DefaultScheduler.lambda$assignResource$8(DefaultScheduler.java:539) ... 38 more Caused by: java.util.concurrent.CompletionException: org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException: Could not acquire the minimum required resources. at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292) at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308) at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:607) at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:591) ... 36 more Caused by: org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException: Could not acquire the minimum required resources. 2 Some time this exception is thrown 2022-10-21 17:38:54 org.apache.flink.runtime.jobmaster.JobMasterException: TaskManager with id 100.114.96.9:39921-bf0a3e is no longer reachable. at org.apache.flink.runtime.jobmaster.JobMaster$TaskManagerHeartbeatListener.notifyTargetUnreachable(JobMaster.java:1359) at org.apache.flink.runtime.heartbeat.HeartbeatMonitorImpl.reportHeartbeatRpcFailure(HeartbeatMonitorImpl.java:123) at org.apache.flink.runtime.heartbeat.HeartbeatManagerImpl.runIfHeartbeatMonitorExists(HeartbeatManagerImpl.java:275) at org.apache.flink.runtime.heartbeat.HeartbeatManagerImpl.reportHeartbeatTargetUnreachable(HeartbeatManagerImpl.java:267) at org.apache.flink.runtime.heartbeat.HeartbeatManagerImpl.handleHeartbeatRpcFailure(HeartbeatManagerImpl.java:262) at org.apache.flink.runtime.heartbeat.HeartbeatManagerImpl.lambda$handleHeartbeatRpc$0(HeartbeatManagerImpl.java:248) at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774) at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750) at java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:456) at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.lambda$handleRunAsync$4(AkkaRpcActor.java:455) at org.apache.flink.runtime.concurrent.akka.ClassLoadingUtils.runWithContextClassLoader(ClassLoadingUtils.java:68) at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRunAsync(AkkaRpcActor.java:455) at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:213) at org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:78) at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:163) at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24) at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20) at scala.PartialFunction.applyOrElse(PartialFunction.scala:123) at scala.PartialFunction.applyOrElse$(PartialFunction.scala:122) at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20) at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171) at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172) at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:172) at akka.actor.Actor.aroundReceive(Actor.scala:537) at akka.actor.Actor.aroundReceive$(Actor.scala:535) at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:220) at akka.actor.ActorCell.receiveMessage(ActorCell.scala:580) at akka.actor.ActorCell.invoke(ActorCell.scala:548) at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:270) at akka.dispatch.Mailbox.run(Mailbox.scala:231) at akka.dispatch.Mailbox.exec(Mailbox.scala:243) at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692) at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
Solr Repeater ReplicationHandler Exception
I am using Solr8.8.0 leader and there are about 16 repeaters. I am seeing following exception on repeater. (Not using SolrCloud) When I checked the Leader instance for file "_9.fdm", it is not there. So my question is where the repeater is getting the name of file from when it does not even exist on Leader? And how to resolve the issue. TIA Exception while writing response for params: generation=11&checksum=true&qt=/replication&file=_9.fdm&wt=filestream&command=filecontent java.nio.file.NoSuchFileException: /app/solr/testCore/data/index.20210817190431551/_9.fdm at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.newFileChannel(UnixFileSystemProvider.java:177) at java.nio.channels.FileChannel.open(FileChannel.java:287) at java.nio.c`enter code here`hannels.FileChannel.open(FileChannel.java:335) at org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:238) at org.apache.lucene.store.NRTCachingDirectory.openInput(NRTCachingDirectory.java:181) at org.apache.solr.handler.ReplicationHandler$DirectoryFileStream.write(ReplicationHandler.java:1648) at org.apache.solr.core.SolrCore$4.write(SolrCore.java:2796) at org.apache.solr.response.QueryResponseWriterUtil.writeQueryResponse(QueryResponseWriterUtil.java:49) at org.apache.solr.servlet.HttpSolrCall.writeResponse(HttpSolrCall.java:890) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:583) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:427) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:357) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:201) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:602) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1612) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1582) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:191) at org.eclipse.jetty.server.handler.InetAccessHandler.handle(InetAccessHandler.java:177) at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:516) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:556) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:273) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:336) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:313) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:129) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:375) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:773) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:905) at java.lang.Thread.run(Thread.java:748)
PyFlink java.io.EOFException at java.io.DataInputStream.readFully
I have a PyFlink job that reads from a file, filter based on a condition, and print. This is a tree view of my working directory. This is the PyFlink script main.py: from pyflink.datastream import StreamExecutionEnvironment from pyflink.table import StreamTableEnvironment, DataTypes from pyflink.table.udf import udf # https://flink.apache.org/2020/04/09/pyflink-udf-support-flink.html # https://ci.apache.org/projects/flink/flink-docs-release-1.12/dev/python/table-api-users-guide/udfs/python_udfs.html #udf(input_types=[DataTypes.STRING()], result_type=DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING())) def parse(s): import json # a dummy parser res = {'item_id': 123, 'tag': 'a'} return res env = StreamExecutionEnvironment.get_execution_environment() t_env = StreamTableEnvironment.create(env) t_env.register_function("parse", parse) my_source_ddl = """ create table mySource ( id BIGINT, contentstr STRING ) with ( 'connector' = 'filesystem', 'format' = 'json', 'path' = '/tmp/input' ) """ my_sink_ddl = """ create table mySink ( id BIGINT ) with ( 'connector' = 'print' ) """ my_transform_dml = """ insert into mySink with t1 as ( select id, parse(contentstr) as content from mySource ) select id from t1 where content['item_id'] is not null and content['tag'] = 'a' """ t_env.execute_sql(my_source_ddl) t_env.execute_sql(my_sink_ddl) t_env.execute_sql(my_transform_dml).wait() To run the main.py: Ensure installing pyflink==1.12.0 in my conda env /tmp/input has a single row of content {"id":1,"tag":"a"} Then I run main.py and I get the exception: Traceback (most recent call last): File "udf_parse.py", line 53, in <module> t_env.execute_sql(my_transform_dml).wait() File "/usr/local/anaconda3/envs/pyflink-quickstart/lib/python3.7/site-packages/pyflink/table/table_result.py", line 76, in wait get_method(self._j_table_result, "await")() File "/usr/local/anaconda3/envs/pyflink-quickstart/lib/python3.7/site-packages/py4j/java_gateway.py", line 1286, in __call__ answer, self.gateway_client, self.target_id, self.name) File "/usr/local/anaconda3/envs/pyflink-quickstart/lib/python3.7/site-packages/pyflink/util/exceptions.py", line 147, in deco return f(*a, **kw) File "/usr/local/anaconda3/envs/pyflink-quickstart/lib/python3.7/site-packages/py4j/protocol.py", line 328, in get_return_value format(target_id, ".", name), value) py4j.protocol.Py4JJavaError: An error occurred while calling o53.await. : java.util.concurrent.ExecutionException: org.apache.flink.table.api.TableException: Failed to wait job finish at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:357) at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1908) at org.apache.flink.table.api.internal.TableResultImpl.awaitInternal(TableResultImpl.java:123) at org.apache.flink.table.api.internal.TableResultImpl.await(TableResultImpl.java:86) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.flink.api.python.shaded.py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) at org.apache.flink.api.python.shaded.py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) at org.apache.flink.api.python.shaded.py4j.Gateway.invoke(Gateway.java:282) at org.apache.flink.api.python.shaded.py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at org.apache.flink.api.python.shaded.py4j.commands.CallCommand.execute(CallCommand.java:79) at org.apache.flink.api.python.shaded.py4j.GatewayConnection.run(GatewayConnection.java:238) at java.lang.Thread.run(Thread.java:748) Caused by: org.apache.flink.table.api.TableException: Failed to wait job finish at org.apache.flink.table.api.internal.InsertResultIterator.hasNext(InsertResultIterator.java:56) at org.apache.flink.table.api.internal.TableResultImpl$CloseableRowIteratorWrapper.hasNext(TableResultImpl.java:350) at org.apache.flink.table.api.internal.TableResultImpl$CloseableRowIteratorWrapper.isFirstRowReady(TableResultImpl.java:363) at org.apache.flink.table.api.internal.TableResultImpl.lambda$awaitInternal$1(TableResultImpl.java:110) at java.util.concurrent.CompletableFuture$AsyncRun.run(CompletableFuture.java:1640) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ... 1 more Caused by: java.util.concurrent.ExecutionException: org.apache.flink.runtime.client.JobExecutionException: Job execution failed. at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:357) at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1908) at org.apache.flink.table.api.internal.InsertResultIterator.hasNext(InsertResultIterator.java:54) ... 7 more Caused by: org.apache.flink.runtime.client.JobExecutionException: Job execution failed. at org.apache.flink.runtime.jobmaster.JobResult.toJobExecutionResult(JobResult.java:144) at org.apache.flink.runtime.minicluster.MiniClusterJobClient.lambda$getJobExecutionResult$2(MiniClusterJobClient.java:117) at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:616) at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:591) at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1975) at org.apache.flink.runtime.rpc.akka.AkkaInvocationHandler.lambda$invokeRpc$0(AkkaInvocationHandler.java:237) at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774) at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750) at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488) at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:1975) at org.apache.flink.runtime.concurrent.FutureUtils$1.onComplete(FutureUtils.java:1046) at akka.dispatch.OnComplete.internal(Future.scala:264) at akka.dispatch.OnComplete.internal(Future.scala:261) at akka.dispatch.japi$CallbackBridge.apply(Future.scala:191) at akka.dispatch.japi$CallbackBridge.apply(Future.scala:188) at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:36) at org.apache.flink.runtime.concurrent.Executors$DirectExecutionContext.execute(Executors.java:73) at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:44) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:252) at akka.pattern.PromiseActorRef.$bang(AskSupport.scala:572) at akka.pattern.PipeToSupport$PipeableFuture$$anonfun$pipeTo$1.applyOrElse(PipeToSupport.scala:22) at akka.pattern.PipeToSupport$PipeableFuture$$anonfun$pipeTo$1.applyOrElse(PipeToSupport.scala:21) at scala.concurrent.Future$$anonfun$andThen$1.apply(Future.scala:436) at scala.concurrent.Future$$anonfun$andThen$1.apply(Future.scala:435) at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:36) at akka.dispatch.BatchingExecutor$AbstractBatch.processBatch(BatchingExecutor.scala:55) at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply$mcV$sp(BatchingExecutor.scala:91) at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply(BatchingExecutor.scala:91) at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply(BatchingExecutor.scala:91) at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72) at akka.dispatch.BatchingExecutor$BlockableBatch.run(BatchingExecutor.scala:90) at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:40) at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:44) at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) Caused by: org.apache.flink.runtime.JobException: Recovery is suppressed by NoRestartBackoffTimeStrategy at org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.handleFailure(ExecutionFailureHandler.java:118) at org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.getFailureHandlingResult(ExecutionFailureHandler.java:80) at org.apache.flink.runtime.scheduler.DefaultScheduler.handleTaskFailure(DefaultScheduler.java:233) at org.apache.flink.runtime.scheduler.DefaultScheduler.maybeHandleTaskFailure(DefaultScheduler.java:224) at org.apache.flink.runtime.scheduler.DefaultScheduler.updateTaskExecutionStateInternal(DefaultScheduler.java:215) at org.apache.flink.runtime.scheduler.SchedulerBase.updateTaskExecutionState(SchedulerBase.java:669) at org.apache.flink.runtime.scheduler.SchedulerNG.updateTaskExecutionState(SchedulerNG.java:89) at org.apache.flink.runtime.jobmaster.JobMaster.updateTaskExecutionState(JobMaster.java:447) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcInvocation(AkkaRpcActor.java:305) at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:212) at org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:77) at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:158) at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:26) at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:21) at scala.PartialFunction$class.applyOrElse(PartialFunction.scala:123) at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:21) at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:170) at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171) at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171) at akka.actor.Actor$class.aroundReceive(Actor.scala:517) at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:225) at akka.actor.ActorCell.receiveMessage(ActorCell.scala:592) at akka.actor.ActorCell.invoke(ActorCell.scala:561) at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:258) at akka.dispatch.Mailbox.run(Mailbox.scala:225) at akka.dispatch.Mailbox.exec(Mailbox.scala:235) ... 4 more Caused by: java.io.EOFException at java.io.DataInputStream.readFully(DataInputStream.java:197) at java.io.DataInputStream.readFully(DataInputStream.java:169) at org.apache.flink.table.runtime.typeutils.StringDataSerializer.deserializeInternal(StringDataSerializer.java:88) at org.apache.flink.table.runtime.typeutils.StringDataSerializer.deserialize(StringDataSerializer.java:82) at org.apache.flink.table.runtime.typeutils.StringDataSerializer.deserialize(StringDataSerializer.java:34) at org.apache.flink.table.runtime.typeutils.serializers.python.MapDataSerializer.deserializeInternal(MapDataSerializer.java:129) at org.apache.flink.table.runtime.typeutils.serializers.python.MapDataSerializer.deserialize(MapDataSerializer.java:110) at org.apache.flink.table.runtime.typeutils.serializers.python.MapDataSerializer.deserialize(MapDataSerializer.java:46) at org.apache.flink.table.runtime.typeutils.serializers.python.RowDataSerializer.deserialize(RowDataSerializer.java:106) at org.apache.flink.table.runtime.typeutils.serializers.python.RowDataSerializer.deserialize(RowDataSerializer.java:49) at org.apache.flink.table.runtime.operators.python.scalar.RowDataPythonScalarFunctionOperator.emitResult(RowDataPythonScalarFunctionOperator.java:81) at org.apache.flink.streaming.api.operators.python.AbstractPythonFunctionOperator.emitResults(AbstractPythonFunctionOperator.java:250) at org.apache.flink.streaming.api.operators.python.AbstractPythonFunctionOperator.invokeFinishBundle(AbstractPythonFunctionOperator.java:273) at org.apache.flink.streaming.api.operators.python.AbstractPythonFunctionOperator.processWatermark(AbstractPythonFunctionOperator.java:199) at org.apache.flink.streaming.runtime.tasks.ChainingOutput.emitWatermark(ChainingOutput.java:123) at org.apache.flink.streaming.runtime.tasks.SourceOperatorStreamTask$AsyncDataOutputToOutput.emitWatermark(SourceOperatorStreamTask.java:170) at org.apache.flink.streaming.runtime.tasks.SourceOperatorStreamTask.advanceToEndOfEventTime(SourceOperatorStreamTask.java:110) at org.apache.flink.streaming.runtime.tasks.SourceOperatorStreamTask.afterInvoke(SourceOperatorStreamTask.java:116) at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:589) at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:755) at org.apache.flink.runtime.taskmanager.Task.run(Task.java:570) at java.lang.Thread.run(Thread.java:748) The issue is probably related to the udf. Any help? Thanks!
The root cause is: #udf(input_types=[DataTypes.STRING()], result_type=DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING())) def parse(s): import json # a dummy parser res = {'item_id': 123, 'tag': 'a'} return res The result_type is DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING()), but item_id has value type integer. A fix is simple: def parse(s): import json # a dummy parser res = {'item_id': 123, 'tag': 'a'} res = {str(k): str(v) for k, v in res.items()} return res
H2 DB Corruption: java.lang.IllegalStateException: Chunk 1936 not found [1.4.200/9]
we are using H2 1.4.200 with Spring Boot 2.2.2 and are experiencing from time to time database chunk corruptions. Therefore the database is not usable anymore. We know that our database is corrupt since recovery leads to: "IllegalStateException: Store header is corrupt". We get the following stack trace on restart: 2020-02-20 08:21:28.900 [main] ERROR com.zaxxer.hikari.pool.HikariPool(595) - HikariPool-1 - Exception during pool initialization. org.h2.jdbc.JdbcSQLNonTransientException: Allgemeiner Fehler: "java.lang.IllegalStateException: Chunk 1936 not found [1.4.200/9]" General error: "java.lang.IllegalStateException: Chunk 1936 not found [1.4.200/9]" [50000-200] at org.h2.message.DbException.getJdbcSQLException(DbException.java:505) at org.h2.message.DbException.getJdbcSQLException(DbException.java:429) at org.h2.message.DbException.get(DbException.java:194) at org.h2.message.DbException.convert(DbException.java:347) at org.h2.engine.Database.openDatabase(Database.java:333) at org.h2.engine.Database.<init>(Database.java:301) at org.h2.engine.Engine.openSession(Engine.java:74) at org.h2.engine.Engine.openSession(Engine.java:192) at org.h2.engine.Engine.createSessionAndValidate(Engine.java:171) at org.h2.engine.Engine.createSession(Engine.java:166) at org.h2.engine.Engine.createSession(Engine.java:29) at org.h2.engine.SessionRemote.connectEmbeddedOrServer(SessionRemote.java:340) at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:173) at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:152) at org.h2.Driver.connect(Driver.java:69) at com.zaxxer.hikari.util.DriverDataSource.getConnection(DriverDataSource.java:138) at com.zaxxer.hikari.pool.PoolBase.newConnection(PoolBase.java:353) at com.zaxxer.hikari.pool.PoolBase.newPoolEntry(PoolBase.java:201) at com.zaxxer.hikari.pool.HikariPool.createPoolEntry(HikariPool.java:473) at com.zaxxer.hikari.pool.HikariPool.checkFailFast(HikariPool.java:562) at com.zaxxer.hikari.pool.HikariPool.<init>(HikariPool.java:115) at com.zaxxer.hikari.HikariDataSource.getConnection(HikariDataSource.java:112) at org.flywaydb.core.internal.jdbc.JdbcUtils.openConnection(JdbcUtils.java:56) at org.flywaydb.core.internal.jdbc.JdbcConnectionFactory.<init>(JdbcConnectionFactory.java:80) at org.flywaydb.core.Flyway.execute(Flyway.java:438) at org.flywaydb.core.Flyway.migrate(Flyway.java:149) at org.springframework.boot.autoconfigure.flyway.FlywayMigrationInitializer.afterPropertiesSet(FlywayMigrationInitializer.java:65) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeInitMethods(AbstractAutowireCapableBeanFactory.java:1855) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1792) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:595) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:517) at org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:323) at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:222) at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:321) at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:202) at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:310) at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:227) at org.springframework.beans.factory.support.DefaultListableBeanFactory.resolveNamedBean(DefaultListableBeanFactory.java:1155) at org.springframework.beans.factory.support.DefaultListableBeanFactory.resolveBean(DefaultListableBeanFactory.java:416) at org.springframework.beans.factory.support.DefaultListableBeanFactory.getBean(DefaultListableBeanFactory.java:349) at org.springframework.beans.factory.support.DefaultListableBeanFactory.getBean(DefaultListableBeanFactory.java:342) at org.springframework.orm.jpa.EntityManagerFactoryUtils.findEntityManagerFactory(EntityManagerFactoryUtils.java:121) at org.springframework.orm.jpa.EntityManagerFactoryAccessor.setBeanFactory(EntityManagerFactoryAccessor.java:155) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeAwareMethods(AbstractAutowireCapableBeanFactory.java:1818) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1783) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:595) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:517) at org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:323) at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:222) at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:321) at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:202) at org.springframework.beans.factory.config.DependencyDescriptor.resolveCandidate(DependencyDescriptor.java:276) at org.springframework.beans.factory.support.DefaultListableBeanFactory.doResolveDependency(DefaultListableBeanFactory.java:1287) at org.springframework.beans.factory.support.DefaultListableBeanFactory.resolveDependency(DefaultListableBeanFactory.java:1207) at org.springframework.beans.factory.support.ConstructorResolver.resolveAutowiredArgument(ConstructorResolver.java:885) at org.springframework.beans.factory.support.ConstructorResolver.createArgumentArray(ConstructorResolver.java:789) at org.springframework.beans.factory.support.ConstructorResolver.instantiateUsingFactoryMethod(ConstructorResolver.java:539) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.instantiateUsingFactoryMethod(AbstractAutowireCapableBeanFactory.java:1338) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBeanInstance(AbstractAutowireCapableBeanFactory.java:1177) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:557) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:517) at org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:323) at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:222) at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:321) at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:202) at org.springframework.beans.factory.config.DependencyDescriptor.resolveCandidate(DependencyDescriptor.java:276) at org.springframework.beans.factory.support.DefaultListableBeanFactory.addCandidateEntry(DefaultListableBeanFactory.java:1503) at org.springframework.beans.factory.support.DefaultListableBeanFactory.findAutowireCandidates(DefaultListableBeanFactory.java:1467) at org.springframework.beans.factory.support.DefaultListableBeanFactory.resolveMultipleBeans(DefaultListableBeanFactory.java:1358) at org.springframework.beans.factory.support.DefaultListableBeanFactory.doResolveDependency(DefaultListableBeanFactory.java:1245) at org.springframework.beans.factory.support.DefaultListableBeanFactory.resolveDependency(DefaultListableBeanFactory.java:1207) at org.springframework.beans.factory.annotation.AutowiredAnnotationBeanPostProcessor$AutowiredMethodElement.inject(AutowiredAnnotationBeanPostProcessor.java:715) at org.springframework.beans.factory.annotation.InjectionMetadata.inject(InjectionMetadata.java:116) at org.springframework.beans.factory.annotation.AutowiredAnnotationBeanPostProcessor.postProcessProperties(AutowiredAnnotationBeanPostProcessor.java:399) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.populateBean(AbstractAutowireCapableBeanFactory.java:1422) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:594) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:517) at org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:323) at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:222) at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:321) at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:202) at org.springframework.beans.factory.support.ConstructorResolver.instantiateUsingFactoryMethod(ConstructorResolver.java:409) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.instantiateUsingFactoryMethod(AbstractAutowireCapableBeanFactory.java:1338) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBeanInstance(AbstractAutowireCapableBeanFactory.java:1177) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:557) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:517) at org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:323) at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:222) at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:321) at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:202) at org.springframework.beans.factory.config.DependencyDescriptor.resolveCandidate(DependencyDescriptor.java:276) at org.springframework.beans.factory.support.DefaultListableBeanFactory.doResolveDependency(DefaultListableBeanFactory.java:1287) at org.springframework.beans.factory.support.DefaultListableBeanFactory.resolveDependency(DefaultListableBeanFactory.java:1207) at org.springframework.beans.factory.annotation.AutowiredAnnotationBeanPostProcessor$AutowiredMethodElement.inject(AutowiredAnnotationBeanPostProcessor.java:715) at org.springframework.beans.factory.annotation.InjectionMetadata.inject(InjectionMetadata.java:116) at org.springframework.beans.factory.annotation.AutowiredAnnotationBeanPostProcessor.postProcessProperties(AutowiredAnnotationBeanPostProcessor.java:399) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.populateBean(AbstractAutowireCapableBeanFactory.java:1422) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:594) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:517) at org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:323) at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:222) at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:321) at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:202) at org.springframework.beans.factory.support.ConstructorResolver.instantiateUsingFactoryMethod(ConstructorResolver.java:409) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.instantiateUsingFactoryMethod(AbstractAutowireCapableBeanFactory.java:1338) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBeanInstance(AbstractAutowireCapableBeanFactory.java:1177) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:557) at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:517) at org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:323) at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:222) at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:321) at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:207) at org.springframework.boot.web.servlet.ServletContextInitializerBeans.getOrderedBeansOfType(ServletContextInitializerBeans.java:211) at org.springframework.boot.web.servlet.ServletContextInitializerBeans.getOrderedBeansOfType(ServletContextInitializerBeans.java:202) at org.springframework.boot.web.servlet.ServletContextInitializerBeans.addServletContextInitializerBeans(ServletContextInitializerBeans.java:96) at org.springframework.boot.web.servlet.ServletContextInitializerBeans.<init>(ServletContextInitializerBeans.java:85) at org.springframework.boot.web.servlet.context.ServletWebServerApplicationContext.getServletContextInitializerBeans(ServletWebServerApplicationContext.java:253) at org.springframework.boot.web.servlet.context.ServletWebServerApplicationContext.selfInitialize(ServletWebServerApplicationContext.java:227) at org.springframework.boot.web.embedded.tomcat.TomcatStarter.onStartup(TomcatStarter.java:53) at org.apache.catalina.core.StandardContext.startInternal(StandardContext.java:5135) at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:183) at org.apache.catalina.core.ContainerBase$StartChild.call(ContainerBase.java:1384) at org.apache.catalina.core.ContainerBase$StartChild.call(ContainerBase.java:1374) at java.base/java.util.concurrent.FutureTask.run(Unknown Source) at org.apache.tomcat.util.threads.InlineExecutorService.execute(InlineExecutorService.java:75) at java.base/java.util.concurrent.AbstractExecutorService.submit(Unknown Source) at org.apache.catalina.core.ContainerBase.startInternal(ContainerBase.java:909) at org.apache.catalina.core.StandardHost.startInternal(StandardHost.java:841) at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:183) at org.apache.catalina.core.ContainerBase$StartChild.call(ContainerBase.java:1384) at org.apache.catalina.core.ContainerBase$StartChild.call(ContainerBase.java:1374) at java.base/java.util.concurrent.FutureTask.run(Unknown Source) at org.apache.tomcat.util.threads.InlineExecutorService.execute(InlineExecutorService.java:75) at java.base/java.util.concurrent.AbstractExecutorService.submit(Unknown Source) at org.apache.catalina.core.ContainerBase.startInternal(ContainerBase.java:909) at org.apache.catalina.core.StandardEngine.startInternal(StandardEngine.java:262) at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:183) at org.apache.catalina.core.StandardService.startInternal(StandardService.java:421) at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:183) at org.apache.catalina.core.StandardServer.startInternal(StandardServer.java:930) at org.apache.catalina.util.LifecycleBase.start(LifecycleBase.java:183) at org.apache.catalina.startup.Tomcat.start(Tomcat.java:459) at org.springframework.boot.web.embedded.tomcat.TomcatWebServer.initialize(TomcatWebServer.java:107) at org.springframework.boot.web.embedded.tomcat.TomcatWebServer.<init>(TomcatWebServer.java:88) at org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory.getTomcatWebServer(TomcatServletWebServerFactory.java:438) at org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory.getWebServer(TomcatServletWebServerFactory.java:191) at org.springframework.boot.web.servlet.context.ServletWebServerApplicationContext.createWebServer(ServletWebServerApplicationContext.java:180) at org.springframework.boot.web.servlet.context.ServletWebServerApplicationContext.onRefresh(ServletWebServerApplicationContext.java:153) at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:544) at org.springframework.boot.web.servlet.context.ServletWebServerApplicationContext.refresh(ServletWebServerApplicationContext.java:141) at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:747) at org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:397) at org.springframework.boot.SpringApplication.run(SpringApplication.java:315) at org.springframework.boot.builder.SpringApplicationBuilder.run(SpringApplicationBuilder.java:140) at com.cenarion.svnplus.app.Application.main(Application.java:79) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) at java.base/java.lang.reflect.Method.invoke(Unknown Source) at org.springframework.boot.loader.MainMethodRunner.run(MainMethodRunner.java:48) at org.springframework.boot.loader.Launcher.launch(Launcher.java:87) at org.springframework.boot.loader.Launcher.launch(Launcher.java:51) at org.springframework.boot.loader.PropertiesLauncher.main(PropertiesLauncher.java:597) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) at java.base/java.lang.reflect.Method.invoke(Unknown Source) at com.exe4j.runtime.LauncherEngine.launch(LauncherEngine.java:85) at com.exe4j.runtime.WinLauncher.main(WinLauncher.java:94) at com.install4j.runtime.launcher.WinLauncher.main(WinLauncher.java:25) Caused by: java.lang.IllegalStateException: Chunk 1936 not found [1.4.200/9] at org.h2.mvstore.DataUtils.newIllegalStateException(DataUtils.java:950) at org.h2.mvstore.MVStore.getChunk(MVStore.java:1230) at org.h2.mvstore.MVStore.readBufferForPage(MVStore.java:1214) at org.h2.mvstore.MVStore.readPage(MVStore.java:2209) at org.h2.mvstore.MVMap.readPage(MVMap.java:672) at org.h2.mvstore.Page$NonLeaf.getChildPage(Page.java:1043) at org.h2.mvstore.Cursor.hasNext(Cursor.java:53) at org.h2.mvstore.MVMap$2$1.hasNext(MVMap.java:802) at org.h2.mvstore.db.LobStorageMap.init(LobStorageMap.java:109) at org.h2.engine.Database.open(Database.java:791) at org.h2.engine.Database.openDatabase(Database.java:307) ... 165 common frames omitted Before the corruption there was also another error which leads to the closing of the database. After the restart we get chunk problem above: 2020-01-31 08:13:49 database: flush org.h2.message.DbException: Allgemeiner Fehler: "java.lang.IllegalStateException: java.lang.NullPointerException [1.4.200/3]" General error: "java.lang.IllegalStateException: java.lang.NullPointerException [1.4.200/3]" [50000-200] at org.h2.message.DbException.get(DbException.java:194) at org.h2.message.DbException.convert(DbException.java:347) at org.h2.mvstore.db.MVTableEngine$1.uncaughtException(MVTableEngine.java:93) at org.h2.mvstore.MVStore.handleException(MVStore.java:2877) at org.h2.mvstore.MVStore.panic(MVStore.java:481) at org.h2.mvstore.MVStore.store(MVStore.java:1338) at org.h2.mvstore.MVStore.store(MVStore.java:1312) at org.h2.mvstore.MVStore.tryCommit(MVStore.java:1272) at org.h2.mvstore.MVStore.writeInBackground(MVStore.java:2780) at org.h2.mvstore.MVStore$BackgroundWriterThread.run(MVStore.java:3290) Caused by: org.h2.jdbc.JdbcSQLNonTransientException: Allgemeiner Fehler: "java.lang.IllegalStateException: java.lang.NullPointerException [1.4.200/3]" General error: "java.lang.IllegalStateException: java.lang.NullPointerException [1.4.200/3]" [50000-200] at org.h2.message.DbException.getJdbcSQLException(DbException.java:505) at org.h2.message.DbException.getJdbcSQLException(DbException.java:429) ... 10 more Caused by: java.lang.IllegalStateException: java.lang.NullPointerException [1.4.200/3] at org.h2.mvstore.DataUtils.newIllegalStateException(DataUtils.java:950) ... 5 more Caused by: java.lang.NullPointerException at org.h2.mvstore.MVStore.acceptChunkOccupancyChanges(MVStore.java:1612) at org.h2.mvstore.MVStore.storeNow(MVStore.java:1433) at org.h2.mvstore.MVStore.store(MVStore.java:1334) ... 4 more 2020-01-31 08:13:50 jdbc[3]: exception org.h2.jdbc.JdbcSQLNonTransientConnectionException: Die Datenbank ist bereits geschlossen The database has been closed [90098-200] at org.h2.message.DbException.getJdbcSQLException(DbException.java:622) at org.h2.message.DbException.getJdbcSQLException(DbException.java:429) at org.h2.message.DbException.get(DbException.java:194) at org.h2.engine.Session.getTransaction(Session.java:1792) at org.h2.engine.Session.startStatementWithinTransaction(Session.java:1815) at org.h2.command.Command.executeQuery(Command.java:190) at org.h2.jdbc.JdbcPreparedStatement.executeQuery(JdbcPreparedStatement.java:114) at com.zaxxer.hikari.pool.ProxyPreparedStatement.executeQuery(ProxyPreparedStatement.java:52) at com.zaxxer.hikari.pool.HikariProxyPreparedStatement.executeQuery(HikariProxyPreparedStatement.java) at org.hibernate.engine.jdbc.internal.ResultSetReturnImpl.extract(ResultSetReturnImpl.java:57) at org.hibernate.loader.plan.exec.internal.AbstractLoadPlanBasedLoader.getResultSet(AbstractLoadPlanBasedLoader.java:390) at org.hibernate.loader.plan.exec.internal.AbstractLoadPlanBasedLoader.executeQueryStatement(AbstractLoadPlanBasedLoader.java:163) at org.hibernate.loader.plan.exec.internal.AbstractLoadPlanBasedLoader.executeLoad(AbstractLoadPlanBasedLoader.java:104) at org.hibernate.loader.entity.plan.AbstractLoadPlanBasedEntityLoader.load(AbstractLoadPlanBasedEntityLoader.java:197) at org.hibernate.persister.entity.AbstractEntityPersister.load(AbstractEntityPersister.java:4350) at org.hibernate.event.internal.DefaultLoadEventListener.loadFromDatasource(DefaultLoadEventListener.java:569) at org.hibernate.event.internal.DefaultLoadEventListener.doLoad(DefaultLoadEventListener.java:537) at org.hibernate.event.internal.DefaultLoadEventListener.load(DefaultLoadEventListener.java:208) at org.hibernate.event.internal.DefaultLoadEventListener.proxyOrLoad(DefaultLoadEventListener.java:332) at org.hibernate.event.internal.DefaultLoadEventListener.doOnLoad(DefaultLoadEventListener.java:108) at org.hibernate.event.internal.DefaultLoadEventListener.onLoad(DefaultLoadEventListener.java:74) at org.hibernate.event.service.internal.EventListenerGroupImpl.fireEventOnEachListener(EventListenerGroupImpl.java:118) at org.hibernate.internal.SessionImpl.fireLoadNoChecks(SessionImpl.java:1168) at org.hibernate.internal.SessionImpl.internalLoad(SessionImpl.java:1033) at org.hibernate.type.EntityType.resolveIdentifier(EntityType.java:687) ... at org.springframework.data.jpa.repository.query.JpaQueryExecution$CollectionExecution.doExecute(JpaQueryExecution.java:126) at org.springframework.data.jpa.repository.query.JpaQueryExecution.execute(JpaQueryExecution.java:88) at org.springframework.data.jpa.repository.query.AbstractJpaQuery.doExecute(AbstractJpaQuery.java:154) at org.springframework.data.jpa.repository.query.AbstractJpaQuery.execute(AbstractJpaQuery.java:142) at org.springframework.data.repository.core.support.RepositoryFactorySupport$QueryExecutorMethodInterceptor.doInvoke(RepositoryFactorySupport.java:618) at org.springframework.data.repository.core.support.RepositoryFactorySupport$QueryExecutorMethodInterceptor.invoke(RepositoryFactorySupport.java:605) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.data.projection.DefaultMethodInvokingMethodInterceptor.invoke(DefaultMethodInvokingMethodInterceptor.java:80) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:366) at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:99) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.dao.support.PersistenceExceptionTranslationInterceptor.invoke(PersistenceExceptionTranslationInterceptor.java:139) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.data.jpa.repository.support.CrudMethodMetadataPostProcessor$CrudMethodMetadataPopulatingMethodInterceptor.invoke(CrudMethodMetadataPostProcessor.java:149) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:93) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:212) at com.sun.proxy.$Proxy166.findDeletedOlderThan(Unknown Source) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) at java.base/java.lang.reflect.Method.invoke(Unknown Source) at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:344) at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:198) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163) at org.springframework.dao.support.PersistenceExceptionTranslationInterceptor.invoke(PersistenceExceptionTranslationInterceptor.java:139) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:93) at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:212) at com.sun.proxy.$Proxy166.findDeletedOlderThan(Unknown Source) at com.cenarion.svnplus.commons.service.GutachtenService.lambda$initBinCleanup$0(GutachtenService.java:82) at com.cenarion.svnplus.commons.security.SecurityContextAwareThreadPoolTaskScheduler.lambda$schedule$3(SecurityContextAwareThreadPoolTaskScheduler.java:91) at org.springframework.scheduling.support.DelegatingErrorHandlingRunnable.run(DelegatingErrorHandlingRunnable.java:54) at org.springframework.scheduling.concurrent.ReschedulingRunnable.run(ReschedulingRunnable.java:93) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.run(Unknown Source) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) Caused by: java.lang.IllegalStateException: java.lang.NullPointerException [1.4.200/3] at org.h2.mvstore.DataUtils.newIllegalStateException(DataUtils.java:950) at org.h2.mvstore.MVStore.store(MVStore.java:1338) at org.h2.mvstore.MVStore.store(MVStore.java:1312) at org.h2.mvstore.MVStore.tryCommit(MVStore.java:1272) at org.h2.mvstore.MVStore.writeInBackground(MVStore.java:2780) at org.h2.mvstore.MVStore$BackgroundWriterThread.run(MVStore.java:3290) Caused by: java.lang.NullPointerException at org.h2.mvstore.MVStore.acceptChunkOccupancyChanges(MVStore.java:1612) at org.h2.mvstore.MVStore.storeNow(MVStore.java:1433) at org.h2.mvstore.MVStore.store(MVStore.java:1334) ... 4 more Can you please help us with this issue?
Another piece in the puzzle: I observe the "chunk xxx not found" message with a corrupted H2 database file sometimes after a delete statement and sometimes after an update statement. Here is some log output: 2021-03-24 11:10:31,752 INFO [153] DELETE FROM MYSCHEMA.MYTABLE WHERE ID = ? LIMIT 1 2021-03-24 11:10:31,760 INFO [153] SELECT ID, X FROM MYSCHEMA.MYTABLE WHERE ID= ? 2021-03-24 11:10:31,800 ERROR [153] org.h2.jdbc.JdbcSQLNonTransientException: Allgemeiner Fehler: "Allgemeiner Fehler: ""java.lang.IllegalStateException: java.lang.NullPointerException [1.4.200/3]"" General error: ""java.lang.IllegalStateException: java.lang.NullPointerException [1.4.200/3]"" [50000-200]" General error: "Allgemeiner Fehler: ""java.lang.IllegalStateException: java.lang.NullPointerException [1.4.200/3]"" General error: ""java.lang.IllegalStateException: java.lang.NullPointerException [1.4.200/3]"" [50000-200]"; SQL statement: ROLLBACK [50000-200] at org.h2.message.DbException.getJdbcSQLException(DbException.java:505) at org.h2.message.DbException.getJdbcSQLException(DbException.java:429) at org.h2.message.DbException.get(DbException.java:194) at org.h2.engine.Database.throwLastBackgroundException(Database.java:2221) at org.h2.engine.Database.commit(Database.java:2202) at org.h2.engine.Session.rollback(Session.java:846) at org.h2.command.dml.TransactionCommand.update(TransactionCommand.java:49) at org.h2.command.CommandContainer.update(CommandContainer.java:198) at org.h2.command.Command.executeUpdate(Command.java:251) at org.h2.jdbc.JdbcConnection.rollbackInternal(JdbcConnection.java:1530) at org.h2.jdbc.JdbcConnection.rollback(JdbcConnection.java:561) at org.h2.jdbcx.JdbcXAConnection.getConnection(JdbcXAConnection.java:106) at org.h2.jdbcx.JdbcConnectionPool.getConnectionNow(JdbcConnectionPool.java:236) at org.h2.jdbcx.JdbcConnectionPool.getConnection(JdbcConnectionPool.java:199) [...] Caused by: org.h2.message.DbException: Allgemeiner Fehler: "java.lang.IllegalStateException: java.lang.NullPointerException [1.4.200/3]" General error: "java.lang.IllegalStateException: java.lang.NullPointerException [1.4.200/3]" [50000-200] at org.h2.message.DbException.get(DbException.java:194) at org.h2.message.DbException.convert(DbException.java:347) at org.h2.mvstore.db.MVTableEngine$1.uncaughtException(MVTableEngine.java:93) at org.h2.mvstore.MVStore.handleException(MVStore.java:2877) at org.h2.mvstore.MVStore.panic(MVStore.java:481) at org.h2.mvstore.MVStore.store(MVStore.java:1338) at org.h2.mvstore.MVStore.store(MVStore.java:1312) at org.h2.mvstore.MVStore.tryCommit(MVStore.java:1272) at org.h2.mvstore.MVStore.writeInBackground(MVStore.java:2780) at org.h2.mvstore.MVStore$BackgroundWriterThread.run(MVStore.java:3290) Caused by: org.h2.jdbc.JdbcSQLNonTransientException: Allgemeiner Fehler: "java.lang.IllegalStateException: java.lang.NullPointerException [1.4.200/3]" General error: "java.lang.IllegalStateException: java.lang.NullPointerException [1.4.200/3]" [50000-200] at org.h2.message.DbException.getJdbcSQLException(DbException.java:505) at org.h2.message.DbException.getJdbcSQLException(DbException.java:429) ... 10 common frames omitted Caused by: java.lang.IllegalStateException: java.lang.NullPointerException [1.4.200/3] at org.h2.mvstore.DataUtils.newIllegalStateException(DataUtils.java:950) ... 5 common frames omitted Caused by: java.lang.NullPointerException: null at org.h2.mvstore.MVStore.acceptChunkOccupancyChanges(MVStore.java:1612) at org.h2.mvstore.MVStore.storeNow(MVStore.java:1433) at org.h2.mvstore.MVStore.store(MVStore.java:1334) ... 4 common frames omitted 2021-03-24 11:10:31,823 ERROR [153] org.h2.jdbc.JdbcSQLNonTransientConnectionException: Die Datenbank ist bereits geschlossen The database has been closed [90098-200] at org.h2.message.DbException.getJdbcSQLException(DbException.java:622) at org.h2.message.DbException.getJdbcSQLException(DbException.java:429) at org.h2.message.DbException.get(DbException.java:194) at org.h2.engine.Session.getTransaction(Session.java:1792) at org.h2.engine.Session.getStatementSavepoint(Session.java:1804) at org.h2.engine.Session.setSavepoint(Session.java:915) at org.h2.command.Command.executeUpdate(Command.java:244) at org.h2.jdbc.JdbcConnection.rollbackInternal(JdbcConnection.java:1530) at org.h2.jdbc.JdbcConnection.rollback(JdbcConnection.java:561) at org.h2.jdbcx.JdbcXAConnection.getConnection(JdbcXAConnection.java:106) at org.h2.jdbcx.JdbcConnectionPool.getConnectionNow(JdbcConnectionPool.java:236) at org.h2.jdbcx.JdbcConnectionPool.getConnection(JdbcConnectionPool.java:199) [...] Caused by: java.lang.IllegalStateException: java.lang.NullPointerException [1.4.200/3] at org.h2.mvstore.DataUtils.newIllegalStateException(DataUtils.java:950) at org.h2.mvstore.MVStore.store(MVStore.java:1338) at org.h2.mvstore.MVStore.store(MVStore.java:1312) at org.h2.mvstore.MVStore.tryCommit(MVStore.java:1272) at org.h2.mvstore.MVStore.writeInBackground(MVStore.java:2780) at org.h2.mvstore.MVStore$BackgroundWriterThread.run(MVStore.java:3290) Caused by: java.lang.NullPointerException: null at org.h2.mvstore.MVStore.acceptChunkOccupancyChanges(MVStore.java:1612) at org.h2.mvstore.MVStore.storeNow(MVStore.java:1433) at org.h2.mvstore.MVStore.store(MVStore.java:1334) ... 4 common frames omitted [...] 2021-03-24 11:10:48,826 ERROR [153] org.h2.jdbc.JdbcSQLNonTransientException: Allgemeiner Fehler: "java.lang.IllegalStateException: Chunk 337 not found [1.4.200/9]" General error: "java.lang.IllegalStateException: Chunk 337 not found [1.4.200/9]" [50000-200] at org.h2.message.DbException.getJdbcSQLException(DbException.java:505) at org.h2.message.DbException.getJdbcSQLException(DbException.java:429) at org.h2.message.DbException.get(DbException.java:194) at org.h2.message.DbException.convert(DbException.java:347) at org.h2.engine.Database.openDatabase(Database.java:333) at org.h2.engine.Database.<init>(Database.java:301) at org.h2.engine.Engine.openSession(Engine.java:74) at org.h2.engine.Engine.openSession(Engine.java:192) at org.h2.engine.Engine.createSessionAndValidate(Engine.java:171) at org.h2.engine.Engine.createSession(Engine.java:166) at org.h2.engine.Engine.createSession(Engine.java:29) at org.h2.engine.SessionRemote.connectEmbeddedOrServer(SessionRemote.java:340) at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:173) at org.h2.jdbc.JdbcConnection.<init>(JdbcConnection.java:152) at org.h2.Driver.connect(Driver.java:69) at org.h2.jdbcx.JdbcDataSource.getJdbcConnection(JdbcDataSource.java:189) at org.h2.jdbcx.JdbcDataSource.getXAConnection(JdbcDataSource.java:352) at org.h2.jdbcx.JdbcDataSource.getPooledConnection(JdbcDataSource.java:384) at org.h2.jdbcx.JdbcConnectionPool.getConnectionNow(JdbcConnectionPool.java:234) at org.h2.jdbcx.JdbcConnectionPool.getConnection(JdbcConnectionPool.java:199) [...] Caused by: java.lang.IllegalStateException: Chunk 337 not found [1.4.200/9] at org.h2.mvstore.DataUtils.newIllegalStateException(DataUtils.java:950) at org.h2.mvstore.MVStore.getChunk(MVStore.java:1230) at org.h2.mvstore.MVStore.readBufferForPage(MVStore.java:1214) at org.h2.mvstore.MVStore.readPage(MVStore.java:2209) at org.h2.mvstore.MVMap.readPage(MVMap.java:672) at org.h2.mvstore.Page$NonLeaf.getChildPage(Page.java:1043) at org.h2.mvstore.Page$NonLeaf.getPrependCursorPos(Page.java:1192) at org.h2.mvstore.Cursor.traverseDown(Cursor.java:157) at org.h2.mvstore.Cursor.<init>(Cursor.java:31) at org.h2.mvstore.Cursor.<init>(Cursor.java:27) at org.h2.mvstore.MVMap$2.iterator(MVMap.java:797) at org.h2.mvstore.db.LobStorageMap.init(LobStorageMap.java:109) at org.h2.engine.Database.open(Database.java:791) at org.h2.engine.Database.openDatabase(Database.java:307) ... 159 common frames omitted
Transaction Roll Back is not working in Spring Data MongoDB
I was testing transaction roll back feature using Spring Data MongoDB(2.2.5) and MongoDB 4.2.5 Community Version. As mentioned in Spring Data official documentation I have configured MongoTransactionManager in the configuration class, and have created a replica set with 3 members in the back end. Also, have used repository for mongo db transactions. Have annotated my Dao method using #Transactional. Even I have implemented the transaction scope in the method, my data is still getting saved in the collections(Person as well Address) and no rollback happening at the Databse layer on a Run time exception in the application. I have provided the required source code and trace from my application. Kindly help to troubleshoot the true and pleas let me if i miss anything from my end. #Slf4j #Configuration #EnableMongoRepositories #EnableTransactionManagement(proxyTargetClass=true) class ApplicationConfig extends AbstractMongoClientConfiguration { #Bean MongoTransactionManager transactionManager(MongoDbFactory dbFactory) { log.info("Creating dbFactory {}", dbFactory); MongoTransactionManager manager=new MongoTransactionManager(dbFactory); log.info("Creating manager {}",manager); return manager; } #Override public MongoClient mongoClient() { return MongoClients.create("mongodb://localhost:27020,localhost:27021,localhost:27022/?replicaSet=rs0"); } #Override protected String getDatabaseName() { return "testDB"; } public #Bean MongoTemplate mongoTemplate() { return new MongoTemplate(mongoClient(), "testDB"); } } #Slf4j #Service("TestDao") public class TestDaoImpl implements TestDao{ #Autowired PersonRepository repository; #Autowired MongoOperations ops; #Autowired AddressRepository addressRepo; #Transactional public void save(List<PersonVO> personList) { //if(!p.getNationaliy().equalsIgnoreCase("AU")) AddressVO addVo=new AddressVO(); addVo.setCity("COCHIN"); addVo.setCountry("IN"); addVo.setHouseNo("Rose Villa 123"); Address addr=AddressMapper.INSTANCE.ToAddress(addVo); addressRepo.save(addr); //ops.insert(addr); log.info("Saving...."); for(PersonVO vo:personList) { log.info("Saving person"); Person p=PersonMapper.INSTANCE.ToPerson(vo); repository.save(p); } throw new RuntimeException(); } } Here's the log, the exception and the stack trace. 2020-04-01 01:21:24.106 INFO 7696 --- [nio-7092-exec-2] o.a.c.c.C.[Tomcat].[localhost].[/] : Initializing Spring DispatcherServlet 'dispatcherServlet' 2020-04-01 01:21:24.106 INFO 7696 --- [nio-7092-exec-2] o.s.web.servlet.DispatcherServlet : Initializing Servlet 'dispatcherServlet' 2020-04-01 01:21:24.127 INFO 7696 --- [nio-7092-exec-2] o.s.web.servlet.DispatcherServlet : Completed initialization in 21 ms 2020-04-01 01:21:24.189 DEBUG 7696 --- [nio-7092-exec-2] o.s.d.mongodb.MongoTransactionManager : Creating new transaction with name [com.example.mongo.demo.service.TestServiceImpl.save]: PROPAGATION_REQUIRED,ISOLATION_DEFAULT 2020-04-01 01:21:24.227 DEBUG 7696 --- [nio-7092-exec-2] o.s.d.mongodb.MongoTransactionManager : About to start transaction for session [ClientSessionImpl#3e82b37d id = {"id": {"$binary": "zw5ZFyfdRPSSOgZ/pW/D3A==", "$type": "04"}}, causallyConsistent = true, txActive = false, txNumber = 0, error = d != java.lang.Boolean]. 2020-04-01 01:21:24.228 DEBUG 7696 --- [nio-7092-exec-2] o.s.d.mongodb.MongoTransactionManager : Started transaction for session [ClientSessionImpl#3e82b37d id = {"id": {"$binary": "zw5ZFyfdRPSSOgZ/pW/D3A==", "$type": "04"}}, causallyConsistent = true, txActive = true, txNumber = 1, error = d != java.lang.Boolean]. 2020-04-01 01:21:24.238 INFO 7696 --- [nio-7092-exec-2] c.e.mongo.demo.service.TestServiceImpl : Saving 2020-04-01 01:21:24.239 DEBUG 7696 --- [nio-7092-exec-2] o.s.d.mongodb.MongoTransactionManager : Participating in existing transaction 2020-04-01 01:21:24.285 DEBUG 7696 --- [nio-7092-exec-2] o.s.data.mongodb.core.MongoTemplate : Inserting Document containing fields: [houseNo, city, country, _class] in collection: address 2020-04-01 01:21:24.334 INFO 7696 --- [nio-7092-exec-2] org.mongodb.driver.connection : Opened connection [connectionId{localValue:7, serverValue:37}] to localhost:27021 2020-04-01 01:21:24.501 INFO 7696 --- [nio-7092-exec-2] com.example.mongo.demo.dao.TestDaoImpl : Saving.... 2020-04-01 01:21:24.501 INFO 7696 --- [nio-7092-exec-2] com.example.mongo.demo.dao.TestDaoImpl : Saving person 2020-04-01 01:21:24.530 DEBUG 7696 --- [nio-7092-exec-2] o.s.data.mongodb.core.MongoTemplate : Inserting Document containing fields: [name, age, gender, nationaliy, _class] in collection: person 2020-04-01 01:21:24.532 INFO 7696 --- [nio-7092-exec-2] com.example.mongo.demo.dao.TestDaoImpl : Saving person 2020-04-01 01:21:24.533 DEBUG 7696 --- [nio-7092-exec-2] o.s.data.mongodb.core.MongoTemplate : Inserting Document containing fields: [name, age, gender, nationaliy, _class] in collection: person 2020-04-01 01:21:24.536 DEBUG 7696 --- [nio-7092-exec-2] o.s.d.mongodb.MongoTransactionManager : Participating transaction failed - marking existing transaction as rollback-only 2020-04-01 01:21:24.536 DEBUG 7696 --- [nio-7092-exec-2] o.s.d.mongodb.MongoTransactionManager : Initiating transaction rollback 2020-04-01 01:21:24.537 DEBUG 7696 --- [nio-7092-exec-2] o.s.d.mongodb.MongoTransactionManager : About to abort transaction for session [ClientSessionImpl#3e82b37d id = {"id": {"$binary": "zw5ZFyfdRPSSOgZ/pW/D3A==", "$type": "04"}}, causallyConsistent = true, txActive = true, txNumber = 1, error = d != java.lang.Boolean]. 2020-04-01 01:21:24.538 DEBUG 7696 --- [nio-7092-exec-2] o.s.d.mongodb.MongoTransactionManager : About to release Session [ClientSessionImpl#3e82b37d id = {"id": {"$binary": "zw5ZFyfdRPSSOgZ/pW/D3A==", "$type": "04"}}, causallyConsistent = true, txActive = false, txNumber = 1, error = d != java.lang.Boolean] after transaction. 2020-04-01 01:21:24.554 ERROR 7696 --- [nio-7092-exec-2] o.a.c.c.C.[.[.[/].[dispatcherServlet] : Servlet.service() for servlet [dispatcherServlet] in context with path [] threw exception [Request processing failed; nested exception is java.lang.RuntimeException] with root cause java.lang.RuntimeException: null at com.example.mongo.demo.dao.TestDaoImpl.save(TestDaoImpl.java:57) ~[classes/:na] at com.example.mongo.demo.dao.TestDaoImpl$$FastClassBySpringCGLIB$$8c1970a4.invoke(<generated>) ~[classes/:na] at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218) ~[spring-core-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:769) ~[spring-aop-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163) ~[spring-aop-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:747) ~[spring-aop-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:366) ~[spring-tx-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:99) ~[spring-tx-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) ~[spring-aop-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:747) ~[spring-aop-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:689) ~[spring-aop-5.2.4.RELEASE.jar:5.2.4.RELEASE] at com.example.mongo.demo.dao.TestDaoImpl$$EnhancerBySpringCGLIB$$8ea2a8.save(<generated>) ~[classes/:na] at com.example.mongo.demo.service.TestServiceImpl.save(TestServiceImpl.java:46) ~[classes/:na] at com.example.mongo.demo.service.TestServiceImpl$$FastClassBySpringCGLIB$$949c2e6a.invoke(<generated>) ~[classes/:na] at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218) ~[spring-core-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:769) ~[spring-aop-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163) ~[spring-aop-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:747) ~[spring-aop-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:366) ~[spring-tx-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:99) ~[spring-tx-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) ~[spring-aop-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:747) ~[spring-aop-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:689) ~[spring-aop-5.2.4.RELEASE.jar:5.2.4.RELEASE] at com.example.mongo.demo.service.TestServiceImpl$$EnhancerBySpringCGLIB$$7980a7ba.save(<generated>) ~[classes/:na] at com.example.mongo.demo.controller.TestController.savePersons(TestController.java:33) ~[classes/:na] at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:na] at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:na] at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:na] at java.base/java.lang.reflect.Method.invoke(Method.java:566) ~[na:na] at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:190) ~[spring-web-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:138) ~[spring-web-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:106) ~[spring-webmvc-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:879) ~[spring-webmvc-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:793) ~[spring-webmvc-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87) ~[spring-webmvc-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1040) ~[spring-webmvc-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:943) ~[spring-webmvc-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006) ~[spring-webmvc-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:898) ~[spring-webmvc-5.2.4.RELEASE.jar:5.2.4.RELEASE] at javax.servlet.http.HttpServlet.service(HttpServlet.java:634) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883) ~[spring-webmvc-5.2.4.RELEASE.jar:5.2.4.RELEASE] at javax.servlet.http.HttpServlet.service(HttpServlet.java:741) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53) ~[tomcat-embed-websocket-9.0.31.jar:9.0.31] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100) ~[spring-web-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) ~[spring-web-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.springframework.web.filter.FormContentFilter.doFilterInternal(FormContentFilter.java:93) ~[spring-web-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) ~[spring-web-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:109) ~[spring-boot-actuator-2.2.5.RELEASE.jar:2.2.5.RELEASE] at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) ~[spring-web-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201) ~[spring-web-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) ~[spring-web-5.2.4.RELEASE.jar:5.2.4.RELEASE] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:202) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:541) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:367) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1639) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[na:na] at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[na:na] at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) ~[tomcat-embed-core-9.0.31.jar:9.0.31] at java.base/java.lang.Thread.run(Thread.java:834) ~[na:na]