I am using GlassFish 3.1.1, SQL Server 2008, and NetBeans with Hibernate 3 and JSF 2.0 to create a web application. I am able to retrieve data with the hibernate API successfully but for some reason, the connection pool becomes null when I try to click a commandButton that takes me from one page to another page. In the action method, I am simply making a call to getOriginCount() but the hibernate API is giving a weird null error. Not sure if it's glassfish pool, hibernate, or sql server.
WARNING: RAR5117 : Failed to obtain/create connection from connection pool [ perfpro-jdbc-pool ]. Reason : com.sun.appserv.connectors.internal.api.PoolingException
WARNING: RAR5114 : Error allocating connection : [Error in allocating a connection. Cause: null]
SEVERE: 1237767 [http-thread-pool-8080(2)] WARN org.hibernate.util.JDBCExceptionReporter - SQL Error: 0, SQLState: null
SEVERE: 1237772 [http-thread-pool-8080(2)] ERROR org.hibernate.util.JDBCExceptionReporter - Error in allocating a connection. Cause: null
WARNING: /cloneBuilderMenu.xhtml #31,168 rendered="#{cloneBuilderBean.canCreateMoreClones}": org.hibernate.exception.GenericJDBCException: Cannot open connection
javax.el.ELException: /cloneBuilderMenu.xhtml #31,168 rendered="#{cloneBuilderBean.canCreateMoreClones}": org.hibernate.exception.GenericJDBCException: Cannot open connection
at com.sun.faces.facelets.el.TagValueExpression.getValue(TagValueExpression.java:114)
at javax.faces.component.ComponentStateHelper.eval(ComponentStateHelper.java:194)
at javax.faces.component.UIComponentBase.isRendered(UIComponentBase.java:413)
at javax.faces.component.UIComponentBase.processDecodes(UIComponentBase.java:1166)
at javax.faces.component.UIForm.processDecodes(UIForm.java:225)
at javax.faces.component.UIComponentBase.processDecodes(UIComponentBase.java:1176)
at javax.faces.component.UIComponentBase.processDecodes(UIComponentBase.java:1176)
at javax.faces.component.UIViewRoot.processDecodes(UIViewRoot.java:933)
at com.sun.faces.lifecycle.ApplyRequestValuesPhase.execute(ApplyRequestValuesPhase.java:78)
at com.sun.faces.lifecycle.Phase.doPhase(Phase.java:101)
at com.sun.faces.lifecycle.LifecycleImpl.execute(LifecycleImpl.java:118)
at javax.faces.webapp.FacesServlet.service(FacesServlet.java:593)
at org.apache.catalina.core.StandardWrapper.service(StandardWrapper.java:1539)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:281)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:175)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:655)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:595)
at com.sun.enterprise.web.WebPipeline.invoke(WebPipeline.java:98)
at com.sun.enterprise.web.PESessionLockingStandardPipeline.invoke(PESessionLockingStandardPipeline.java:91)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:162)
at org.apache.catalina.connector.CoyoteAdapter.doService(CoyoteAdapter.java:330)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:231)
at com.sun.enterprise.v3.services.impl.ContainerMapper.service(ContainerMapper.java:174)
at com.sun.grizzly.http.ProcessorTask.invokeAdapter(ProcessorTask.java:828)
at com.sun.grizzly.http.ProcessorTask.doProcess(ProcessorTask.java:725)
at com.sun.grizzly.http.ProcessorTask.process(ProcessorTask.java:1019)
at com.sun.grizzly.http.DefaultProtocolFilter.execute(DefaultProtocolFilter.java:225)
at com.sun.grizzly.DefaultProtocolChain.executeProtocolFilter(DefaultProtocolChain.java:137)
at com.sun.grizzly.DefaultProtocolChain.execute(DefaultProtocolChain.java:104)
at com.sun.grizzly.DefaultProtocolChain.execute(DefaultProtocolChain.java:90)
at com.sun.grizzly.http.HttpProtocolChain.execute(HttpProtocolChain.java:79)
at com.sun.grizzly.ProtocolChainContextTask.doCall(ProtocolChainContextTask.java:54)
at com.sun.grizzly.SelectionKeyContextTask.call(SelectionKeyContextTask.java:59)
at com.sun.grizzly.ContextTask.run(ContextTask.java:71)
at com.sun.grizzly.util.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:532)
at com.sun.grizzly.util.AbstractThreadPool$Worker.run(AbstractThreadPool.java:513)
at java.lang.Thread.run(Thread.java:662)
Caused by: org.hibernate.exception.GenericJDBCException: Cannot open connection
at org.hibernate.exception.SQLStateConverter.handledNonSpecificException(SQLStateConverter.java:140)
at org.hibernate.exception.SQLStateConverter.convert(SQLStateConverter.java:128)
at org.hibernate.exception.JDBCExceptionHelper.convert(JDBCExceptionHelper.java:66)
at org.hibernate.exception.JDBCExceptionHelper.convert(JDBCExceptionHelper.java:52)
at org.hibernate.jdbc.ConnectionManager.openConnection(ConnectionManager.java:449)
at org.hibernate.jdbc.ConnectionManager.getConnection(ConnectionManager.java:167)
at org.hibernate.jdbc.JDBCContext.connection(JDBCContext.java:160)
at org.hibernate.transaction.JDBCTransaction.begin(JDBCTransaction.java:81)
at org.hibernate.impl.SessionImpl.beginTransaction(SessionImpl.java:1473)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.hibernate.context.ThreadLocalSessionContext$TransactionProtectionWrapper.invoke(ThreadLocalSessionContext.java:344)
at $Proxy146.beginTransaction(Unknown Source)
at com.perpro.controller.dao.impl.AssessmentDaoImpl.getOriginCount(AssessmentDaoImpl.java:145)
at com.perpro.controller.cloneBuilderBean.getCanCreateMoreClones(cloneBuilderBean.java:248)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at javax.el.BeanELResolver.getValue(BeanELResolver.java:302)
at com.sun.faces.el.DemuxCompositeELResolver._getValue(DemuxCompositeELResolver.java:176)
at com.sun.faces.el.DemuxCompositeELResolver.getValue(DemuxCompositeELResolver.java:203)
at com.sun.el.parser.AstValue.getValue(AstValue.java:116)
at com.sun.el.parser.AstValue.getValue(AstValue.java:163)
at com.sun.el.ValueExpressionImpl.getValue(ValueExpressionImpl.java:219)
at org.jboss.weld.el.WeldValueExpression.getValue(WeldValueExpression.java:55)
at com.sun.faces.facelets.el.TagValueExpression.getValue(TagValueExpression.java:109)
... 36 more
Caused by: java.sql.SQLException: Error in allocating a connection. Cause: null
at com.sun.gjc.spi.base.DataSource.getConnection(DataSource.java:120)
at org.hibernate.connection.DatasourceConnectionProvider.getConnection(DatasourceConnectionProvider.java:92)
at org.hibernate.jdbc.ConnectionManager.openConnection(ConnectionManager.java:446)
... 60 more
Caused by: javax.resource.spi.ResourceAllocationException: Error in allocating a connection. Cause: null
at com.sun.enterprise.connectors.ConnectionManagerImpl.internalGetConnection(ConnectionManagerImpl.java:307)
at com.sun.enterprise.connectors.ConnectionManagerImpl.allocateConnection(ConnectionManagerImpl.java:190)
at com.sun.enterprise.connectors.ConnectionManagerImpl.allocateConnection(ConnectionManagerImpl.java:165)
at com.sun.enterprise.connectors.ConnectionManagerImpl.allocateConnection(ConnectionManagerImpl.java:160)
at com.sun.gjc.spi.base.DataSource.getConnection(DataSource.java:113)
... 62 more
Caused by: com.sun.appserv.connectors.internal.api.PoolingException
at com.sun.enterprise.resource.pool.datastructure.RWLockDataStructure.addResource(RWLockDataStructure.java:103)
at com.sun.enterprise.resource.pool.ConnectionPool.addResource(ConnectionPool.java:282)
at com.sun.enterprise.resource.pool.ConnectionPool.createResourceAndAddToPool(ConnectionPool.java:1501)
at com.sun.enterprise.resource.pool.ConnectionPool.createResources(ConnectionPool.java:944)
at com.sun.enterprise.resource.pool.ConnectionPool.resizePoolAndGetNewResource(ConnectionPool.java:792)
at com.sun.enterprise.resource.pool.ConnectionPool.getResourceFromPool(ConnectionPool.java:760)
at com.sun.enterprise.resource.pool.ConnectionPool.getUnenlistedResource(ConnectionPool.java:632)
at com.sun.enterprise.resource.pool.ConnectionPool.internalGetResource(ConnectionPool.java:526)
at com.sun.enterprise.resource.pool.ConnectionPool.getResource(ConnectionPool.java:381)
at com.sun.enterprise.resource.pool.PoolManagerImpl.getResourceFromPool(PoolManagerImpl.java:242)
at com.sun.enterprise.resource.pool.PoolManagerImpl.getResource(PoolManagerImpl.java:167)
at com.sun.enterprise.connectors.ConnectionManagerImpl.getResource(ConnectionManagerImpl.java:338)
at com.sun.enterprise.connectors.ConnectionManagerImpl.internalGetConnection(ConnectionManagerImpl.java:301)
... 66 more
Caused by: com.sun.appserv.connectors.internal.api.PoolingException
at com.sun.enterprise.resource.pool.ConnectionPool.createSingleResource(ConnectionPool.java:924)
at com.sun.enterprise.resource.pool.ConnectionPool.createResource(ConnectionPool.java:1185)
at com.sun.enterprise.resource.pool.datastructure.RWLockDataStructure.addResource(RWLockDataStructure.java:98)
... 78 more
Caused by: java.lang.NullPointerException
at com.microsoft.sqlserver.jdbc.FailoverMapSingleton.getFailoverInfo(FailOverMapSingleton.java:37)
at com.microsoft.sqlserver.jdbc.SQLServerConnection.connect(SQLServerConnection.java:685)
at com.microsoft.sqlserver.jdbc.SQLServerDataSource.getConnectionInternal(SQLServerDataSource.java:593)
at com.microsoft.sqlserver.jdbc.SQLServerDataSource.getConnection(SQLServerDataSource.java:57)
at com.sun.gjc.spi.DSManagedConnectionFactory.createManagedConnection(DSManagedConnectionFactory.java:113)
at com.sun.enterprise.resource.allocator.LocalTxConnectorAllocator.createResource(LocalTxConnectorAllocator.java:87)
at com.sun.enterprise.resource.pool.ConnectionPool.createSingleResource(ConnectionPool.java:907)
... 80 more
WARNING: RAR5058: Error while Resizing pool perfpro-jdbc-pool. Exception : null
WARNING: RAR5058: Error while Resizing pool perfpro-jdbc-pool. Exception : null
WARNING: RAR5058: Error while Resizing pool perfpro-jdbc-pool. Exception : null
WARNING: RAR5058: Error while Resizing pool perfpro-jdbc-pool. Exception : null
WARNING: RAR5058: Error while Resizing pool perfpro-jdbc-pool. Exception : null
WARNING: RAR5058: Error while Resizing pool perfpro-jdbc-pool. Exception : null
WARNING: RAR5058: Error while Resizing pool perfpro-jdbc-pool. Exception : null
Here is the method where the nullpointer happens at session.beginTransaction(); The HibernateUtil class is generated by Netbeans.
public Integer getOriginCount() throws Exception {
Session session = HibernateUtil.getSessionFactory().getCurrentSession();
session.beginTransaction();
Integer originCount = ((Long) session.createQuery("select count(*) from origin").uniqueResult()).intValue();
session.getTransaction().commit();
return originCount;
}
The NPE seem to originate from a SQLServerDataSource.getConnection call. It is possible that either your datasource or database is incorrectly configured. To start with, I will try disabling failover clustering.
Related
When I executed the docker run command, it's throwing me an error. Tables are created in the database but throwing errors.
docker run -p 8080:8080 -e DB_VENDOR=MYSQL -e DB_ADDR=192.168.18.01 -e DB_PORT=3306 -e DB_USER=keycloak -e DB_PASSWORD=keycloak -e KEYCLOAK_USER=admin -e KEYCLOAK_PASSWORD=admin -e JDBC_PARAMS="useSSL=false&allowPublicKeyRetrieval=true" -e JAVA_OPTS="-Djboss.as.management.blocking.timeout=10800" -e KEYCLOAK_LOGLEVEL=DEBUG -v "D:/test":/opt/jboss/test keycloak/keycloak:latest
Error
ERROR [org.keycloak.connections.jpa.updater.liquibase.LiquibaseJpaUpdaterProvider] (ServerService Thread Pool -- 61) Error has occurred while updating the database: liquibase.exception.MigrationFailedException: Migration failed for change set META-INF/jpa-changelog-1.2.0.CR1.xml::1.2.0.RC1::bburke#redhat.com:
Reason: liquibase.exception.UnexpectedLiquibaseException: java.sql.SQLException: IJ031070: Transaction cannot proceed: STATUS_ROLLEDBACK
at org.liquibase//liquibase.changelog.ChangeSet.execute(ChangeSet.java:607)
at org.liquibase//liquibase.changelog.visitor.UpdateVisitor.visit(UpdateVisitor.java:51)
at org.liquibase//liquibase.changelog.ChangeLogIterator.run(ChangeLogIterator.java:79)
at org.liquibase//liquibase.Liquibase.update(Liquibase.java:214)
at org.liquibase//liquibase.Liquibase.update(Liquibase.java:192)
at org.liquibase//liquibase.Liquibase.update(Liquibase.java:188)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.updater.liquibase.LiquibaseJpaUpdaterProvider.updateChangeSet(LiquibaseJpaUpdaterProvider.java:183)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.updater.liquibase.LiquibaseJpaUpdaterProvider.update(LiquibaseJpaUpdaterProvider.java:102)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.updater.liquibase.LiquibaseJpaUpdaterProvider.update(LiquibaseJpaUpdaterProvider.java:81)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.DefaultJpaConnectionProviderFactory$2.run(DefaultJpaConnectionProviderFactory.java:341)
at org.keycloak.keycloak-server-spi-private#12.0.4//org.keycloak.models.utils.KeycloakModelUtils.runJobInTransaction(KeycloakModelUtils.java:228)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.DefaultJpaConnectionProviderFactory.update(DefaultJpaConnectionProviderFactory.java:334)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.DefaultJpaConnectionProviderFactory.migration(DefaultJpaConnectionProviderFactory.java:306)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.DefaultJpaConnectionProviderFactory.lambda$lazyInit$0(DefaultJpaConnectionProviderFactory.java:182)
at org.keycloak.keycloak-server-spi-private#12.0.4//org.keycloak.models.utils.KeycloakModelUtils.suspendJtaTransaction(KeycloakModelUtils.java:654)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.DefaultJpaConnectionProviderFactory.lazyInit(DefaultJpaConnectionProviderFactory.java:133)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.DefaultJpaConnectionProviderFactory.create(DefaultJpaConnectionProviderFactory.java:81)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.DefaultJpaConnectionProviderFactory.create(DefaultJpaConnectionProviderFactory.java:59)
at org.keycloak.keycloak-services#12.0.4//org.keycloak.services.DefaultKeycloakSession.getProvider(DefaultKeycloakSession.java:274)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.models.jpa.JpaRealmProviderFactory.create(JpaRealmProviderFactory.java:51)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.models.jpa.JpaRealmProviderFactory.create(JpaRealmProviderFactory.java:33)
at org.keycloak.keycloak-services#12.0.4//org.keycloak.services.DefaultKeycloakSession.getProvider(DefaultKeycloakSession.java:274)
at org.keycloak.keycloak-services#12.0.4//org.keycloak.services.DefaultKeycloakSession.realmLocalStorage(DefaultKeycloakSession.java:199)
at org.keycloak.keycloak-model-infinispan#12.0.4//org.keycloak.models.cache.infinispan.RealmCacheSession.getRealmDelegate(RealmCacheSession.java:152)
at org.keycloak.keycloak-model-infinispan#12.0.4//org.keycloak.models.cache.infinispan.RealmCacheSession.getMigrationModel(RealmCacheSession.java:145)
at org.keycloak.keycloak-server-spi-private#12.0.4//org.keycloak.migration.MigrationModelManager.migrate(MigrationModelManager.java:99)
at org.keycloak.keycloak-services#12.0.4//org.keycloak.services.resources.KeycloakApplication.migrateModel(KeycloakApplication.java:234)
at org.keycloak.keycloak-services#12.0.4//org.keycloak.services.resources.KeycloakApplication.migrateAndBootstrap(KeycloakApplication.java:175)
at org.keycloak.keycloak-services#12.0.4//org.keycloak.services.resources.KeycloakApplication$1.run(KeycloakApplication.java:138)
at org.keycloak.keycloak-server-spi-private#12.0.4//org.keycloak.models.utils.KeycloakModelUtils.runJobInTransaction(KeycloakModelUtils.java:228)
at org.keycloak.keycloak-services#12.0.4//org.keycloak.services.resources.KeycloakApplication.startup(KeycloakApplication.java:129)
at org.keycloak.keycloak-wildfly-extensions#12.0.4//org.keycloak.provider.wildfly.WildflyPlatform.onStartup(WildflyPlatform.java:29)
at org.keycloak.keycloak-services#12.0.4//org.keycloak.services.resources.KeycloakApplication.<init>(KeycloakApplication.java:115)
at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:490)
at org.jboss.resteasy.resteasy-jaxrs#3.13.2.Final//org.jboss.resteasy.core.ConstructorInjectorImpl.construct(ConstructorInjectorImpl.java:152)
at org.jboss.resteasy.resteasy-jaxrs#3.13.2.Final//org.jboss.resteasy.spi.ResteasyProviderFactory.createProviderInstance(ResteasyProviderFactory.java:2815)
at org.jboss.resteasy.resteasy-jaxrs#3.13.2.Final//org.jboss.resteasy.spi.ResteasyDeployment.createApplication(ResteasyDeployment.java:371)
at org.jboss.resteasy.resteasy-jaxrs#3.13.2.Final//org.jboss.resteasy.spi.ResteasyDeployment.startInternal(ResteasyDeployment.java:283)
at org.jboss.resteasy.resteasy-jaxrs#3.13.2.Final//org.jboss.resteasy.spi.ResteasyDeployment.start(ResteasyDeployment.java:93)
at org.jboss.resteasy.resteasy-jaxrs#3.13.2.Final//org.jboss.resteasy.plugins.server.servlet.ServletContainerDispatcher.init(ServletContainerDispatcher.java:140)
at org.jboss.resteasy.resteasy-jaxrs#3.13.2.Final//org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.init(HttpServletDispatcher.java:42)
at io.undertow.servlet#2.2.2.Final//io.undertow.servlet.core.LifecyleInterceptorInvocation.proceed(LifecyleInterceptorInvocation.java:117)
at org.wildfly.extension.undertow#21.0.2.Final//org.wildfly.extension.undertow.security.RunAsLifecycleInterceptor.init(RunAsLifecycleInterceptor.java:78)
at io.undertow.servlet#2.2.2.Final//io.undertow.servlet.core.LifecyleInterceptorInvocation.proceed(LifecyleInterceptorInvocation.java:103)
at io.undertow.servlet#2.2.2.Final//io.undertow.servlet.core.ManagedServlet$DefaultInstanceStrategy.start(ManagedServlet.java:305)
at io.undertow.servlet#2.2.2.Final//io.undertow.servlet.core.ManagedServlet.createServlet(ManagedServlet.java:145)
at io.undertow.servlet#2.2.2.Final//io.undertow.servlet.core.DeploymentManagerImpl$2.call(DeploymentManagerImpl.java:588)
at io.undertow.servlet#2.2.2.Final//io.undertow.servlet.core.DeploymentManagerImpl$2.call(DeploymentManagerImpl.java:559)
at io.undertow.servlet#2.2.2.Final//io.undertow.servlet.core.ServletRequestContextThreadSetupAction$1.call(ServletRequestContextThreadSetupAction.java:42)
at io.undertow.servlet#2.2.2.Final//io.undertow.servlet.core.ContextClassLoaderSetupAction$1.call(ContextClassLoaderSetupAction.java:43)
at org.wildfly.extension.undertow#21.0.2.Final//org.wildfly.extension.undertow.security.SecurityContextThreadSetupAction.lambda$create$0(SecurityContextThreadSetupAction.java:105)
at org.wildfly.extension.undertow#21.0.2.Final//org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1530)
at org.wildfly.extension.undertow#21.0.2.Final//org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1530)
at org.wildfly.extension.undertow#21.0.2.Final//org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1530)
at org.wildfly.extension.undertow#21.0.2.Final//org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1530)
at io.undertow.servlet#2.2.2.Final//io.undertow.servlet.core.DeploymentManagerImpl.start(DeploymentManagerImpl.java:601)
at org.wildfly.extension.undertow#21.0.2.Final//org.wildfly.extension.undertow.deployment.UndertowDeploymentService.startContext(UndertowDeploymentService.java:97)
at org.wildfly.extension.undertow#21.0.2.Final//org.wildfly.extension.undertow.deployment.UndertowDeploymentService$1.run(UndertowDeploymentService.java:78)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at org.jboss.threads#2.4.0.Final//org.jboss.threads.ContextClassLoaderSavingRunnable.run(ContextClassLoaderSavingRunnable.java:35)
at org.jboss.threads#2.4.0.Final//org.jboss.threads.EnhancedQueueExecutor.safeRun(EnhancedQueueExecutor.java:1990)
at org.jboss.threads#2.4.0.Final//org.jboss.threads.EnhancedQueueExecutor$ThreadBody.doRunTask(EnhancedQueueExecutor.java:1486)
at org.jboss.threads#2.4.0.Final//org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1377)
at java.base/java.lang.Thread.run(Thread.java:829)
at org.jboss.threads#2.4.0.Final//org.jboss.threads.JBossThread.run(JBossThread.java:513)
Caused by: liquibase.exception.UnexpectedLiquibaseException: java.sql.SQLException: IJ031070: Transaction cannot proceed: STATUS_ROLLEDBACK
at org.liquibase//liquibase.database.jvm.JdbcConnection.getURL(JdbcConnection.java:79)
at org.liquibase//liquibase.executor.jvm.JdbcExecutor.execute(JdbcExecutor.java:66)
at org.liquibase//liquibase.executor.jvm.JdbcExecutor.execute(JdbcExecutor.java:113)
at org.liquibase//liquibase.database.AbstractJdbcDatabase.execute(AbstractJdbcDatabase.java:1277)
at org.liquibase//liquibase.database.AbstractJdbcDatabase.executeStatements(AbstractJdbcDatabase.java:1259)
at org.liquibase//liquibase.changelog.ChangeSet.execute(ChangeSet.java:582)
... 68 more
Caused by: java.sql.SQLException: IJ031070: Transaction cannot proceed: STATUS_ROLLEDBACK
at org.jboss.ironjacamar.jdbcadapters#1.4.23.Final//org.jboss.jca.adapters.jdbc.WrapperDataSource.checkTransactionActive(WrapperDataSource.java:272)
at org.jboss.ironjacamar.jdbcadapters#1.4.23.Final//org.jboss.jca.adapters.jdbc.WrappedConnection.checkTransactionActive(WrappedConnection.java:2007)
at org.jboss.ironjacamar.jdbcadapters#1.4.23.Final//org.jboss.jca.adapters.jdbc.WrappedConnection.checkStatus(WrappedConnection.java:2022)
at org.jboss.ironjacamar.jdbcadapters#1.4.23.Final//org.jboss.jca.adapters.jdbc.WrappedConnection.checkTransaction(WrappedConnection.java:1996)
at org.jboss.ironjacamar.jdbcadapters#1.4.23.Final//org.jboss.jca.adapters.jdbc.WrappedConnection.getMetaData(WrappedConnection.java:982)
at org.liquibase//liquibase.database.jvm.JdbcConnection.getURL(JdbcConnection.java:77)
... 73 more
Error during startup: java.lang.RuntimeException: Failed to update database
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.updater.liquibase.LiquibaseJpaUpdaterProvider.update(LiquibaseJpaUpdaterProvider.java:117)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.updater.liquibase.LiquibaseJpaUpdaterProvider.update(LiquibaseJpaUpdaterProvider.java:81)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.DefaultJpaConnectionProviderFactory$2.run(DefaultJpaConnectionProviderFactory.java:341)
at org.keycloak.keycloak-server-spi-private#12.0.4//org.keycloak.models.utils.KeycloakModelUtils.runJobInTransaction(KeycloakModelUtils.java:228)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.DefaultJpaConnectionProviderFactory.update(DefaultJpaConnectionProviderFactory.java:334)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.DefaultJpaConnectionProviderFactory.migration(DefaultJpaConnectionProviderFactory.java:306)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.DefaultJpaConnectionProviderFactory.lambda$lazyInit$0(DefaultJpaConnectionProviderFactory.java:182)
at org.keycloak.keycloak-server-spi-private#12.0.4//org.keycloak.models.utils.KeycloakModelUtils.suspendJtaTransaction(KeycloakModelUtils.java:654)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.DefaultJpaConnectionProviderFactory.lazyInit(DefaultJpaConnectionProviderFactory.java:133)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.DefaultJpaConnectionProviderFactory.create(DefaultJpaConnectionProviderFactory.java:81)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.DefaultJpaConnectionProviderFactory.create(DefaultJpaConnectionProviderFactory.java:59)
at org.keycloak.keycloak-services#12.0.4//org.keycloak.services.DefaultKeycloakSession.getProvider(DefaultKeycloakSession.java:274)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.models.jpa.JpaRealmProviderFactory.create(JpaRealmProviderFactory.java:51)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.models.jpa.JpaRealmProviderFactory.create(JpaRealmProviderFactory.java:33)
at org.keycloak.keycloak-services#12.0.4//org.keycloak.services.DefaultKeycloakSession.getProvider(DefaultKeycloakSession.java:274)
at org.keycloak.keycloak-services#12.0.4//org.keycloak.services.DefaultKeycloakSession.realmLocalStorage(DefaultKeycloakSession.java:199)
at org.keycloak.keycloak-model-infinispan#12.0.4//org.keycloak.models.cache.infinispan.RealmCacheSession.getRealmDelegate(RealmCacheSession.java:152)
at org.keycloak.keycloak-model-infinispan#12.0.4//org.keycloak.models.cache.infinispan.RealmCacheSession.getMigrationModel(RealmCacheSession.java:145)
at org.keycloak.keycloak-server-spi-private#12.0.4//org.keycloak.migration.MigrationModelManager.migrate(MigrationModelManager.java:99)
at org.keycloak.keycloak-services#12.0.4//org.keycloak.services.resources.KeycloakApplication.migrateModel(KeycloakApplication.java:234)
at org.keycloak.keycloak-services#12.0.4//org.keycloak.services.resources.KeycloakApplication.migrateAndBootstrap(KeycloakApplication.java:175)
at org.keycloak.keycloak-services#12.0.4//org.keycloak.services.resources.KeycloakApplication$1.run(KeycloakApplication.java:138)
at org.keycloak.keycloak-server-spi-private#12.0.4//org.keycloak.models.utils.KeycloakModelUtils.runJobInTransaction(KeycloakModelUtils.java:228)
at org.keycloak.keycloak-services#12.0.4//org.keycloak.services.resources.KeycloakApplication.startup(KeycloakApplication.java:129)
at org.keycloak.keycloak-wildfly-extensions#12.0.4//org.keycloak.provider.wildfly.WildflyPlatform.onStartup(WildflyPlatform.java:29)
at org.keycloak.keycloak-services#12.0.4//org.keycloak.services.resources.KeycloakApplication.<init>(KeycloakApplication.java:115)
at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:490)
at org.jboss.resteasy.resteasy-jaxrs#3.13.2.Final//org.jboss.resteasy.core.ConstructorInjectorImpl.construct(ConstructorInjectorImpl.java:152)
at org.jboss.resteasy.resteasy-jaxrs#3.13.2.Final//org.jboss.resteasy.spi.ResteasyProviderFactory.createProviderInstance(ResteasyProviderFactory.java:2815)
at org.jboss.resteasy.resteasy-jaxrs#3.13.2.Final//org.jboss.resteasy.spi.ResteasyDeployment.createApplication(ResteasyDeployment.java:371)
at org.jboss.resteasy.resteasy-jaxrs#3.13.2.Final//org.jboss.resteasy.spi.ResteasyDeployment.startInternal(ResteasyDeployment.java:283)
at org.jboss.resteasy.resteasy-jaxrs#3.13.2.Final//org.jboss.resteasy.spi.ResteasyDeployment.start(ResteasyDeployment.java:93)
at org.jboss.resteasy.resteasy-jaxrs#3.13.2.Final//org.jboss.resteasy.plugins.server.servlet.ServletContainerDispatcher.init(ServletContainerDispatcher.java:140)
at org.jboss.resteasy.resteasy-jaxrs#3.13.2.Final//org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.init(HttpServletDispatcher.java:42)
at io.undertow.servlet#2.2.2.Final//io.undertow.servlet.core.LifecyleInterceptorInvocation.proceed(LifecyleInterceptorInvocation.java:117)
at org.wildfly.extension.undertow#21.0.2.Final//org.wildfly.extension.undertow.security.RunAsLifecycleInterceptor.init(RunAsLifecycleInterceptor.java:78)
at io.undertow.servlet#2.2.2.Final//io.undertow.servlet.core.LifecyleInterceptorInvocation.proceed(LifecyleInterceptorInvocation.java:103)
at io.undertow.servlet#2.2.2.Final//io.undertow.servlet.core.ManagedServlet$DefaultInstanceStrategy.start(ManagedServlet.java:305)
at io.undertow.servlet#2.2.2.Final//io.undertow.servlet.core.ManagedServlet.createServlet(ManagedServlet.java:145)
at io.undertow.servlet#2.2.2.Final//io.undertow.servlet.core.DeploymentManagerImpl$2.call(DeploymentManagerImpl.java:588)
at io.undertow.servlet#2.2.2.Final//io.undertow.servlet.core.DeploymentManagerImpl$2.call(DeploymentManagerImpl.java:559)
at io.undertow.servlet#2.2.2.Final//io.undertow.servlet.core.ServletRequestContextThreadSetupAction$1.call(ServletRequestContextThreadSetupAction.java:42)
at io.undertow.servlet#2.2.2.Final//io.undertow.servlet.core.ContextClassLoaderSetupAction$1.call(ContextClassLoaderSetupAction.java:43)
at org.wildfly.extension.undertow#21.0.2.Final//org.wildfly.extension.undertow.security.SecurityContextThreadSetupAction.lambda$create$0(SecurityContextThreadSetupAction.java:105)
at org.wildfly.extension.undertow#21.0.2.Final//org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1530)
at org.wildfly.extension.undertow#21.0.2.Final//org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1530)
at org.wildfly.extension.undertow#21.0.2.Final//org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1530)
at org.wildfly.extension.undertow#21.0.2.Final//org.wildfly.extension.undertow.deployment.UndertowDeploymentInfoService$UndertowThreadSetupAction.lambda$create$0(UndertowDeploymentInfoService.java:1530)
at io.undertow.servlet#2.2.2.Final//io.undertow.servlet.core.DeploymentManagerImpl.start(DeploymentManagerImpl.java:601)
at org.wildfly.extension.undertow#21.0.2.Final//org.wildfly.extension.undertow.deployment.UndertowDeploymentService.startContext(UndertowDeploymentService.java:97)
at org.wildfly.extension.undertow#21.0.2.Final//org.wildfly.extension.undertow.deployment.UndertowDeploymentService$1.run(UndertowDeploymentService.java:78)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at org.jboss.threads#2.4.0.Final//org.jboss.threads.ContextClassLoaderSavingRunnable.run(ContextClassLoaderSavingRunnable.java:35)
at org.jboss.threads#2.4.0.Final//org.jboss.threads.EnhancedQueueExecutor.safeRun(EnhancedQueueExecutor.java:1990)
at org.jboss.threads#2.4.0.Final//org.jboss.threads.EnhancedQueueExecutor$ThreadBody.doRunTask(EnhancedQueueExecutor.java:1486)
at org.jboss.threads#2.4.0.Final//org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1377)
at java.base/java.lang.Thread.run(Thread.java:829)
at org.jboss.threads#2.4.0.Final//org.jboss.threads.JBossThread.run(JBossThread.java:513)
Caused by: liquibase.exception.MigrationFailedException: Migration failed for change set META-INF/jpa-changelog-1.2.0.CR1.xml::1.2.0.RC1::bburke#redhat.com:
Reason: liquibase.exception.UnexpectedLiquibaseException: java.sql.SQLException: IJ031070: Transaction cannot proceed: STATUS_ROLLEDBACK
at org.liquibase//liquibase.changelog.ChangeSet.execute(ChangeSet.java:607)
at org.liquibase//liquibase.changelog.visitor.UpdateVisitor.visit(UpdateVisitor.java:51)
at org.liquibase//liquibase.changelog.ChangeLogIterator.run(ChangeLogIterator.java:79)
at org.liquibase//liquibase.Liquibase.update(Liquibase.java:214)
at org.liquibase//liquibase.Liquibase.update(Liquibase.java:192)
at org.liquibase//liquibase.Liquibase.update(Liquibase.java:188)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.updater.liquibase.LiquibaseJpaUpdaterProvider.updateChangeSet(LiquibaseJpaUpdaterProvider.java:183)
at org.keycloak.keycloak-model-jpa#12.0.4//org.keycloak.connections.jpa.updater.liquibase.LiquibaseJpaUpdaterProvider.update(LiquibaseJpaUpdaterProvider.java:102)
... 61 more
Caused by: liquibase.exception.UnexpectedLiquibaseException: java.sql.SQLException: IJ031070: Transaction cannot proceed: STATUS_ROLLEDBACK
at org.liquibase//liquibase.database.jvm.JdbcConnection.getURL(JdbcConnection.java:79)
at org.liquibase//liquibase.executor.jvm.JdbcExecutor.execute(JdbcExecutor.java:66)
at org.liquibase//liquibase.executor.jvm.JdbcExecutor.execute(JdbcExecutor.java:113)
at org.liquibase//liquibase.database.AbstractJdbcDatabase.execute(AbstractJdbcDatabase.java:1277)
at org.liquibase//liquibase.database.AbstractJdbcDatabase.executeStatements(AbstractJdbcDatabase.java:1259)
at org.liquibase//liquibase.changelog.ChangeSet.execute(ChangeSet.java:582)
... 68 more
Caused by: java.sql.SQLException: IJ031070: Transaction cannot proceed: STATUS_ROLLEDBACK
at org.jboss.ironjacamar.jdbcadapters#1.4.23.Final//org.jboss.jca.adapters.jdbc.WrapperDataSource.checkTransactionActive(WrapperDataSource.java:272)
at org.jboss.ironjacamar.jdbcadapters#1.4.23.Final//org.jboss.jca.adapters.jdbc.WrappedConnection.checkTransactionActive(WrappedConnection.java:2007)
at org.jboss.ironjacamar.jdbcadapters#1.4.23.Final//org.jboss.jca.adapters.jdbc.WrappedConnection.checkStatus(WrappedConnection.java:2022)
at org.jboss.ironjacamar.jdbcadapters#1.4.23.Final//org.jboss.jca.adapters.jdbc.WrappedConnection.checkTransaction(WrappedConnection.java:1996)
at org.jboss.ironjacamar.jdbcadapters#1.4.23.Final//org.jboss.jca.adapters.jdbc.WrappedConnection.getMetaData(WrappedConnection.java:982)
at org.liquibase//liquibase.database.jvm.JdbcConnection.getURL(JdbcConnection.java:77)
... 73 more
ERROR [org.jboss.as.controller.management-operation] (Controller Boot Thread) WFLYCTL0013: Operation ("add") failed - address: ([("subsystem" => "microprofile-metrics-smallrye")]): java.lang.NullPointerException
at org.wildfly.extension.microprofile.metrics-smallrye#21.0.2.Final//org.wildfly.extension.microprofile.metrics.MicroProfileMetricsSubsystemAdd$2.execute(MicroProfileMetricsSubsystemAdd.java:89)
at org.jboss.as.controller#13.0.3.Final//org.jboss.as.controller.AbstractOperationContext.executeStep(AbstractOperationContext.java:1005)
at org.jboss.as.controller#13.0.3.Final//org.jboss.as.controller.AbstractOperationContext.processStages(AbstractOperationContext.java:744)
at org.jboss.as.controller#13.0.3.Final//org.jboss.as.controller.AbstractOperationContext.executeOperation(AbstractOperationContext.java:468)
at org.jboss.as.controller#13.0.3.Final//org.jboss.as.controller.OperationContextImpl.executeOperation(OperationContextImpl.java:1415)
at org.jboss.as.controller#13.0.3.Final//org.jboss.as.controller.ModelControllerImpl.boot(ModelControllerImpl.java:529)
at org.jboss.as.controller#13.0.3.Final//org.jboss.as.controller.AbstractControllerService.boot(AbstractControllerService.java:515)
at org.jboss.as.controller#13.0.3.Final//org.jboss.as.controller.AbstractControllerService.boot(AbstractControllerService.java:477)
at org.jboss.as.server#13.0.3.Final//org.jboss.as.server.ServerService.boot(ServerService.java:451)
at org.jboss.as.server#13.0.3.Final//org.jboss.as.server.ServerService.boot(ServerService.java:404)
at org.jboss.as.controller#13.0.3.Final//org.jboss.as.controller.AbstractControllerService$1.run(AbstractControllerService.java:416)
at java.base/java.lang.Thread.run(Thread.java:829)
07:37:33,800 ERROR [org.jboss.as.server] (ServerService Thread Pool -- 46) WFLYSRV0022: Deploy of deployment "keycloak-server.war" was rolled back with no failure message
Your MySQL server does not send a RSA public key. In this case you need to allow the mysql client to query it from the server with an extra round-trip.
You need to enable allowPublicKeyRetrieval.
Set your JDBC_PARAMS to useSSL=false&allowPublicKeyRetrieval=true for this to work.
Note: Please don't do this in production and enable SSL instead.
I'm running my cluster on kubernetes with a single jobmanager and 2 taskmanagers.
I tested the mechanism of checkpoint by killing one of the taskmanager pods while the job is running.
I got the following exceptions on the jobmanager and the restarted taskmanager:
Jobmanager exception:
java.lang.Exception: Exception while creating StreamOperatorStateContext.
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:195)
at org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:253)
at org.apache.flink.streaming.runtime.tasks.StreamTask.initializeState(StreamTask.java:881)
at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:395)
at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:705)
at org.apache.flink.runtime.taskmanager.Task.run(Task.java:530)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.flink.util.FlinkException: Could not restore keyed state backend for WindowOperator_54288f79b169ee3e8cb1feb33bbad4c3_(1/8) from any of the 1 provided restore options.
at org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:135)
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:307)
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:135)
... 6 more
Caused by: org.apache.flink.runtime.state.BackendBuildingException: Caught unexpected exception.
at org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:326)
at org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:520)
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:291)
at org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142)
at org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121)
... 8 more
Caused by: java.nio.file.NoSuchFileException: /rocksdb/job_0a1a61f5cbecc09fbaef1257b3392b3a_op_WindowOperator_54288f79b169ee3e8cb1feb33bbad4c3__1_8__uuid_8b95eb2f-f6cf-4c35-8274-a9055376163d/db/000021.sst -> /rocksdb/job_0a1a61f5cbecc09fbaef1257b3392b3a_op_WindowOperator_54288f79b169ee3e8cb1feb33bbad4c3__1_8__uuid_8b95eb2f-f6cf-4c35-8274-a9055376163d/f1a97117-3810-400e-85ca-6e8c998a5ed4/000021.sst
at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86)
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
at sun.nio.fs.UnixFileSystemProvider.createLink(UnixFileSystemProvider.java:476)
at java.nio.file.Files.createLink(Files.java:1086)
at org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation.restoreInstanceDirectoryFromPath(RocksDBIncrementalRestoreOperation.java:473)
at org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation.restoreFromLocalState(RocksDBIncrementalRestoreOperation.java:212)
at org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation.restoreFromRemoteState(RocksDBIncrementalRestoreOperation.java:188)
at org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation.restoreWithoutRescaling(RocksDBIncrementalRestoreOperation.java:162)
at org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation.restore(RocksDBIncrementalRestoreOperation.java:148)
at org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:270)
... 12 more
Taskmanager exception:
2020-01-13 09:26:01,943 ERROR org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder - Caught unexpected exception.
org.apache.flink.fs.s3base.shaded.com.amazonaws.SdkClientException: Failed to sanitize XML document destined for handler class org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.model.transform.XmlResponsesSaxParser$ListBucketHandler
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.model.transform.XmlResponsesSaxParser.sanitizeXmlDocument(XmlResponsesSaxParser.java:219)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.model.transform.XmlResponsesSaxParser.parseListBucketObjectsResponse(XmlResponsesSaxParser.java:317)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.model.transform.Unmarshallers$ListObjectsUnmarshaller.unmarshall(Unmarshallers.java:70)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.model.transform.Unmarshallers$ListObjectsUnmarshaller.unmarshall(Unmarshallers.java:59)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.internal.S3XmlResponseHandler.handle(S3XmlResponseHandler.java:62)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.internal.S3XmlResponseHandler.handle(S3XmlResponseHandler.java:31)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.response.AwsResponseHandlerAdapter.handle(AwsResponseHandlerAdapter.java:70)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleResponse(AmazonHttpClient.java:1554)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1272)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1056)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:743)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4325)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4272)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4266)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.AmazonS3Client.listObjects(AmazonS3Client.java:834)
at org.apache.flink.fs.s3presto.shaded.com.facebook.presto.hive.s3.PrestoS3FileSystem.listPrefix(PrestoS3FileSystem.java:484)
at org.apache.flink.fs.s3presto.shaded.com.facebook.presto.hive.s3.PrestoS3FileSystem.access$000(PrestoS3FileSystem.java:112)
at org.apache.flink.fs.s3presto.shaded.com.facebook.presto.hive.s3.PrestoS3FileSystem$1.<init>(PrestoS3FileSystem.java:271)
at org.apache.flink.fs.s3presto.shaded.com.facebook.presto.hive.s3.PrestoS3FileSystem.listLocatedStatus(PrestoS3FileSystem.java:269)
at org.apache.flink.fs.s3presto.shaded.com.facebook.presto.hive.s3.PrestoS3FileSystem.listStatus(PrestoS3FileSystem.java:258)
at org.apache.flink.fs.s3.common.hadoop.HadoopFileSystem.listStatus(HadoopFileSystem.java:157)
at org.apache.flink.core.fs.SafetyNetWrapperFileSystem.listStatus(SafetyNetWrapperFileSystem.java:97)
at org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation.restoreInstanceDirectoryFromPath(RocksDBIncrementalRestoreOperation.java:460)
at org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation.restoreFromLocalState(RocksDBIncrementalRestoreOperation.java:212)
at org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation.restoreFromRemoteState(RocksDBIncrementalRestoreOperation.java:188)
at org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation.restoreWithoutRescaling(RocksDBIncrementalRestoreOperation.java:162)
at org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation.restore(RocksDBIncrementalRestoreOperation.java:148)
at org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:270)
at org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:520)
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:291)
at org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142)
at org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121)
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:307)
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:135)
at org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:253)
at org.apache.flink.streaming.runtime.tasks.StreamTask.initializeState(StreamTask.java:881)
at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:395)
at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:705)
at org.apache.flink.runtime.taskmanager.Task.run(Task.java:530)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.flink.fs.s3base.shaded.com.amazonaws.AbortedException:
at org.apache.flink.fs.s3base.shaded.com.amazonaws.internal.SdkFilterInputStream.abortIfNeeded(SdkFilterInputStream.java:53)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:81)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.event.ProgressInputStream.read(ProgressInputStream.java:180)
at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284)
at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326)
at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178)
at java.io.InputStreamReader.read(InputStreamReader.java:184)
at java.io.BufferedReader.read1(BufferedReader.java:210)
at java.io.BufferedReader.read(BufferedReader.java:286)
at java.io.Reader.read(Reader.java:140)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.model.transform.XmlResponsesSaxParser.sanitizeXmlDocument(XmlResponsesSaxParser.java:191)
... 44 more
2020-01-13 09:26:01,944 WARN org.apache.flink.streaming.api.operators.BackendRestorerProcedure - Exception while restoring keyed state backend for WindowOperator_54288f79b169ee3e8cb1feb33bbad4c3_(7/8) from alternative (1/1), will retry while more alternatives are available.
org.apache.flink.runtime.state.BackendBuildingException: Caught unexpected exception.
at org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:326)
at org.apache.flink.contrib.streaming.state.RocksDBStateBackend.createKeyedStateBackend(RocksDBStateBackend.java:520)
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.lambda$keyedStatedBackend$1(StreamTaskStateInitializerImpl.java:291)
at org.apache.flink.streaming.api.operators.BackendRestorerProcedure.attemptCreateAndRestore(BackendRestorerProcedure.java:142)
at org.apache.flink.streaming.api.operators.BackendRestorerProcedure.createAndRestore(BackendRestorerProcedure.java:121)
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.keyedStatedBackend(StreamTaskStateInitializerImpl.java:307)
at org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl.streamOperatorStateContext(StreamTaskStateInitializerImpl.java:135)
at org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:253)
at org.apache.flink.streaming.runtime.tasks.StreamTask.initializeState(StreamTask.java:881)
at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:395)
at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:705)
at org.apache.flink.runtime.taskmanager.Task.run(Task.java:530)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.flink.fs.s3base.shaded.com.amazonaws.SdkClientException: Failed to sanitize XML document destined for handler class org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.model.transform.XmlResponsesSaxParser$ListBucketHandler
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.model.transform.XmlResponsesSaxParser.sanitizeXmlDocument(XmlResponsesSaxParser.java:219)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.model.transform.XmlResponsesSaxParser.parseListBucketObjectsResponse(XmlResponsesSaxParser.java:317)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.model.transform.Unmarshallers$ListObjectsUnmarshaller.unmarshall(Unmarshallers.java:70)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.model.transform.Unmarshallers$ListObjectsUnmarshaller.unmarshall(Unmarshallers.java:59)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.internal.S3XmlResponseHandler.handle(S3XmlResponseHandler.java:62)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.internal.S3XmlResponseHandler.handle(S3XmlResponseHandler.java:31)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.response.AwsResponseHandlerAdapter.handle(AwsResponseHandlerAdapter.java:70)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleResponse(AmazonHttpClient.java:1554)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1272)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1056)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:743)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:717)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:699)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:667)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:649)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:513)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4325)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4272)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:4266)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.AmazonS3Client.listObjects(AmazonS3Client.java:834)
at org.apache.flink.fs.s3presto.shaded.com.facebook.presto.hive.s3.PrestoS3FileSystem.listPrefix(PrestoS3FileSystem.java:484)
at org.apache.flink.fs.s3presto.shaded.com.facebook.presto.hive.s3.PrestoS3FileSystem.access$000(PrestoS3FileSystem.java:112)
at org.apache.flink.fs.s3presto.shaded.com.facebook.presto.hive.s3.PrestoS3FileSystem$1.<init>(PrestoS3FileSystem.java:271)
at org.apache.flink.fs.s3presto.shaded.com.facebook.presto.hive.s3.PrestoS3FileSystem.listLocatedStatus(PrestoS3FileSystem.java:269)
at org.apache.flink.fs.s3presto.shaded.com.facebook.presto.hive.s3.PrestoS3FileSystem.listStatus(PrestoS3FileSystem.java:258)
at org.apache.flink.fs.s3.common.hadoop.HadoopFileSystem.listStatus(HadoopFileSystem.java:157)
at org.apache.flink.core.fs.SafetyNetWrapperFileSystem.listStatus(SafetyNetWrapperFileSystem.java:97)
at org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation.restoreInstanceDirectoryFromPath(RocksDBIncrementalRestoreOperation.java:460)
at org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation.restoreFromLocalState(RocksDBIncrementalRestoreOperation.java:212)
at org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation.restoreFromRemoteState(RocksDBIncrementalRestoreOperation.java:188)
at org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation.restoreWithoutRescaling(RocksDBIncrementalRestoreOperation.java:162)
at org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation.restore(RocksDBIncrementalRestoreOperation.java:148)
at org.apache.flink.contrib.streaming.state.RocksDBKeyedStateBackendBuilder.build(RocksDBKeyedStateBackendBuilder.java:270)
... 12 more
Caused by: org.apache.flink.fs.s3base.shaded.com.amazonaws.AbortedException:
at org.apache.flink.fs.s3base.shaded.com.amazonaws.internal.SdkFilterInputStream.abortIfNeeded(SdkFilterInputStream.java:53)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.internal.SdkFilterInputStream.read(SdkFilterInputStream.java:81)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.event.ProgressInputStream.read(ProgressInputStream.java:180)
at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:284)
at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:326)
at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:178)
at java.io.InputStreamReader.read(InputStreamReader.java:184)
at java.io.BufferedReader.read1(BufferedReader.java:210)
at java.io.BufferedReader.read(BufferedReader.java:286)
at java.io.Reader.read(Reader.java:140)
at org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.model.transform.XmlResponsesSaxParser.sanitizeXmlDocument(XmlResponsesSaxParser.java:191)
... 44 more
When I tried to restore from a savepoint, everything works as expected.
Any idea?
From our experience, one possible cause could be you come across below exception first:
Caused by: org.apache.flink.fs.s3base.shaded.com.amazonaws.SdkClientException: Failed to sanitize XML document destined for handler class org.apache.flink.fs.s3base.shaded.com.amazonaws.services.s3.model.transform.XmlResponsesSaxParser$ListBucketHandler
Then the procedure to restore rocksdb state-backend would be interrupted leading to file /rocksdb/job_0a1a61f5cbecc09fbaef1257b3392b3a_op_WindowOperator_54288f79b169ee3e8cb1feb33bbad4c3__1_8__uuid_8b95eb2f-f6cf-4c35-8274-a9055376163d/f1a97117-3810-400e-85ca-6e8c998a5ed4/000021.sst deleted in https://github.com/apache/flink/blob/390926e61aeb69837c70a024ad6e7ff02eccdf2d/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/contrib/streaming/state/restore/RocksDBIncrementalRestoreOperation.java#L197
That's why you found the NoSuchFileException.
I tried to apply jpa transaction manager with spring transaction policy to one route. As a result, it is starting and thrown below issue. could you please let me know the cause of the issue and solution.Could you please share any quick-starts of camel jpa transaction for standalone mode. I can find quick-starts for fuse and server mode.
[pache.camel.spring.Main.main()] DefaultTypeConverter INFO Loaded 240 type converters
[pache.camel.spring.Main.main()] DefaultRuntimeEndpointRegistry INFO Runtime endpoint registry is in extended mode gathering usage statistics of all incoming and outgoing endpoints (cache limit: 1000)
[pache.camel.spring.Main.main()] JpaComponent INFO Using EntityManagerFactory configured: org.springframework.orm.jpa.LocalEntityManagerFactoryBean#147d849
[pache.camel.spring.Main.main()] JpaComponent INFO Using TransactionManager configured on this component: org.springframework.orm.jpa.JpaTransactionManager#a9b98d
[ERROR] *************************************
[ERROR] Error occurred while running main from: org.apache.camel.spring.Main
[ERROR]
java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
at org.apache.camel.maven.RunMojo$1.run(RunMojo.java:458)
at java.lang.Thread.run(Thread.java:744)
Caused by: java.lang.NoSuchMethodError: org.apache.camel.processor.RedeliveryErrorHandler.<init>(Lorg/apache/camel/CamelContext;Lorg/apache/camel/Processor;Lorg/apache/camel/util/CamelLogger;Lorg/apache/camel/Processor;Lorg/apache/camel/processor/RedeliveryPolicy;Lorg/apache/camel/Processor;Ljava/lang/String;ZZLorg/apache/camel/Predicate;Ljava/util/concurrent/ScheduledExecutorService;Lorg/apache/camel/Processor;)V
at org.apache.camel.spring.spi.TransactionErrorHandler.<init>(TransactionErrorHandler.java:70)
at org.apache.camel.spring.spi.TransactionErrorHandlerBuilder.createErrorHandler(TransactionErrorHandlerBuilder.java:110)
at org.apache.camel.spring.spi.SpringTransactionPolicy.createTransactionErrorHandler(SpringTransactionPolicy.java:124)
at org.apache.camel.spring.spi.SpringTransactionPolicy.wrap(SpringTransactionPolicy.java:108)
at org.apache.camel.model.TransactedDefinition.createProcessor(TransactedDefinition.java:162)
at org.apache.camel.model.ProcessorDefinition.makeProcessorImpl(ProcessorDefinition.java:534)
at org.apache.camel.model.ProcessorDefinition.makeProcessor(ProcessorDefinition.java:495)
at org.apache.camel.model.ProcessorDefinition.addRoutes(ProcessorDefinition.java:219)
at org.apache.camel.model.RouteDefinition.addRoutes(RouteDefinition.java:1069)
at org.apache.camel.model.RouteDefinition.addRoutes(RouteDefinition.java:196)
at org.apache.camel.impl.DefaultCamelContext.startRoute(DefaultCamelContext.java:974)
at org.apache.camel.impl.DefaultCamelContext.startRouteDefinitions(DefaultCamelContext.java:3301)
at org.apache.camel.impl.DefaultCamelContext.doStartCamel(DefaultCamelContext.java:3024)
at org.apache.camel.impl.DefaultCamelContext.access$000(DefaultCamelContext.java:175)
at org.apache.camel.impl.DefaultCamelContext$2.call(DefaultCamelContext.java:2854)
at org.apache.camel.impl.DefaultCamelContext$2.call(DefaultCamelContext.java:2850)
at org.apache.camel.impl.DefaultCamelContext.doWithDefinedClassLoader(DefaultCamelContext.java:2873)
at org.apache.camel.impl.DefaultCamelContext.doStart(DefaultCamelContext.java:2850)
at org.apache.camel.support.ServiceSupport.start(ServiceSupport.java:61)
at org.apache.camel.impl.DefaultCamelContext.start(DefaultCamelContext.java:2819)
at org.apache.camel.spring.SpringCamelContext.maybeStart(SpringCamelContext.java:270)
at org.apache.camel.spring.SpringCamelContext.onApplicationEvent(SpringCamelContext.java:136)
at org.apache.camel.spring.CamelContextFactoryBean.onApplicationEvent(CamelContextFactoryBean.java:340)
at org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:96)
at org.springframework.context.support.AbstractApplicationContext.publishEvent(AbstractApplicationContext.java:334)
at org.springframework.context.support.AbstractApplicationContext.finishRefresh(AbstractApplicationContext.java:954)
at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:482)
at org.springframework.context.support.FileSystemXmlApplicationContext.<init>(FileSystemXmlApplicationContext.java:140)
at org.springframework.context.support.FileSystemXmlApplicationContext.<init>(FileSystemXmlApplicationContext.java:94)
at org.apache.camel.spring.Main.createDefaultApplicationContext(Main.java:205)
at org.apache.camel.spring.Main.doStart(Main.java:154)
at org.apache.camel.support.ServiceSupport.start(ServiceSupport.java:61)
at org.apache.camel.main.MainSupport.run(MainSupport.java:138)
at org.apache.camel.main.MainSupport.run(MainSupport.java:390)
at org.apache.camel.spring.Main.main(Main.java:87)
... 6 more
[ERROR] *************************************
[WARNING] thread Thread[Timer-0,5,org.apache.camel.spring.Main] was interrupted but is still alive after waiting at least 15000msecs
[WARNING] thread Thread[Timer-0,5,org.apache.camel.spring.Main] will linger despite being asked to die via interruption
[WARNING] thread Thread[derby.rawStoreDaemon,5,derby.daemons] will linger despite being asked to die via interruption
[WARNING] NOTE: 2 thread(s) did not finish despite being asked to via interruption. This is not a problem with exec:java, it is a problem with the running code. Although not serious, it should be remedied.
[WARNING] Couldn't destroy threadgroup org.apache.camel.maven.RunMojo$IsolatedThreadGroup[name=org.apache.camel.spring.Main,maxpri=10]
java.lang.IllegalThreadStateException
at java.lang.ThreadGroup.destroy(ThreadGroup.java:778)
at org.apache.camel.maven.RunMojo.execute(RunMojo.java:491)
at org.apache.maven.plugin.DefaultBuildPluginManager.executeMojo(DefaultBuildPluginManager.java:137)
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:208)
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:154)
at org.apache.maven.lifecycle.internal.MojoExecutor.execute(MojoExecutor.java:146)
at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:117)
at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject(LifecycleModuleBuilder.java:81)
at org.apache.maven.lifecycle.internal.builder.singlethreaded.SingleThreadedBuilder.build(SingleThreadedBuilder.java:56)
at org.apache.maven.lifecycle.internal.LifecycleStarter.execute(LifecycleStarter.java:128)
at org.apache.maven.DefaultMaven.doExecute(DefaultMaven.java:305)
at org.apache.maven.DefaultMaven.doExecute(DefaultMaven.java:192)
at org.apache.maven.DefaultMaven.execute(DefaultMaven.java:105)
at org.apache.maven.cli.MavenCli.execute(MavenCli.java:956)
at org.apache.maven.cli.MavenCli.doMain(MavenCli.java:290)
at org.apache.maven.cli.MavenCli.main(MavenCli.java:194)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
at org.codehaus.plexus.classworlds.launcher.Launcher.launchEnhanced(Launcher.java:289)
at org.codehaus.plexus.classworlds.launcher.Launcher.launch(Launcher.java:229)
at org.codehaus.plexus.classworlds.launcher.Launcher.mainWithExitCode(Launcher.java:415)
at org.codehaus.plexus.classworlds.launcher.Launcher.main(Launcher.java:356)
[INFO] ------------------------------------------------------------------------
From time to time we get Internal Server errors when we push to Gerrit. In the logs we see that the connection the connection is closed before the transaction is completed. (See log below)
We're using an Oracle DB with the following settings:
[database]
type = ORACLE
poolmaxidle = 16
database = db/ReviewDB
port = 1521
instance = infrapd1
hostname = HOSTNAME
connectionpool = true
username = USERNAME
poollimit = 32
dataSourceInterceptorClass = com.googlesource.gerrit.plugins.javamelody.MonitoringDataSourceInterceptor
In the logs we see errors like these when we get an Internal Server error:
[2015-02-13 15:49:51,505] ERROR com.google.gerrit.server.git.ReceiveCommits : Can't insert change/patchset for PROJECT
com.google.gerrit.server.git.InsertException: Error inserting change/patchset
at com.google.gerrit.server.git.ReceiveCommits$1.apply(ReceiveCommits.java:258)
at com.google.gerrit.server.git.ReceiveCommits$1.apply(ReceiveCommits.java:249)
at com.google.common.util.concurrent.Futures$MappingCheckedFuture.mapException(Futures.java:1672)
at com.google.common.util.concurrent.AbstractCheckedFuture.checkedGet(AbstractCheckedFuture.java:85)
at com.google.gerrit.server.git.ReceiveCommits.insertChangesAndPatchSets(ReceiveCommits.java:756)
at com.google.gerrit.server.git.ReceiveCommits.processCommands(ReceiveCommits.java:564)
at com.google.gerrit.server.git.AsyncReceiveCommits$Worker.run(AsyncReceiveCommits.java:90)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at com.google.gerrit.server.util.RequestScopePropagator$5.call(RequestScopePropagator.java:222)
at com.google.gerrit.server.util.RequestScopePropagator$4.call(RequestScopePropagator.java:201)
at com.google.gerrit.server.util.ThreadLocalRequestScopePropagator$1.call(ThreadLocalRequestScopePropagator.java:55)
at com.google.gerrit.server.util.RequestScopePropagator$1.call(RequestScopePropagator.java:98)
at com.google.gerrit.server.util.RequestScopePropagator$2.run(RequestScopePropagator.java:131)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
at com.google.gerrit.server.git.WorkQueue$Task.run(WorkQueue.java:364)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
Caused by: java.util.concurrent.ExecutionException: com.google.gwtorm.server.OrmException: Cannot set auto commit mode
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
at java.util.concurrent.FutureTask.get(FutureTask.java:188)
at com.google.common.util.concurrent.ForwardingFuture.get(ForwardingFuture.java:63)
at com.google.common.util.concurrent.AbstractCheckedFuture.checkedGet(AbstractCheckedFuture.java:78)
... 17 more
Caused by: com.google.gwtorm.server.OrmException: Cannot set auto commit mode
at com.google.gwtorm.jdbc.JdbcSchema.rollback(JdbcSchema.java:80)
at com.google.gerrit.server.change.ChangeInserter.insert(ChangeInserter.java:177)
at com.google.gerrit.server.git.ReceiveCommits$CreateRequest.insertChange(ReceiveCommits.java:1585)
at com.google.gerrit.server.git.ReceiveCommits$CreateRequest.access$800(ReceiveCommits.java:1520)
at com.google.gerrit.server.git.ReceiveCommits$CreateRequest$1.call(ReceiveCommits.java:1551)
at com.google.gerrit.server.git.ReceiveCommits$CreateRequest$1.call(ReceiveCommits.java:1547)
at com.google.gerrit.server.util.RequestScopePropagator$1.call(RequestScopePropagator.java:96)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at com.google.common.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:297)
at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:132)
at com.google.common.util.concurrent.AbstractListeningExecutorService.submit(AbstractListeningExecutorService.java:58)
at com.google.gerrit.server.git.ReceiveCommits$CreateRequest.insertChange(ReceiveCommits.java:1546)
at com.google.gerrit.server.git.ReceiveCommits.insertChangesAndPatchSets(ReceiveCommits.java:752)
... 16 more
Caused by: java.sql.SQLRecoverableException: Closed Connection
at oracle.jdbc.driver.PhysicalConnection.setAutoCommit(PhysicalConnection.java:3790)
at org.apache.commons.dbcp.DelegatingConnection.setAutoCommit(DelegatingConnection.java:371)
at org.apache.commons.dbcp.PoolingDataSource$PoolGuardConnectionWrapper.setAutoCommit(PoolingDataSource.java:328)
at sun.reflect.GeneratedMethodAccessor69.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at net.bull.javamelody.JdbcWrapper$ConnectionInvocationHandler.invoke(JdbcWrapper.java:188)
at net.bull.javamelody.JdbcWrapper$DelegatingInvocationHandler.invoke(JdbcWrapper.java:285)
at com.sun.proxy.$Proxy17.setAutoCommit(Unknown Source)
at com.google.gwtorm.jdbc.JdbcSchema.rollback(JdbcSchema.java:78)
... 28 more
Any suggestions on how to resolve this? Any advise on how to improve the DB settings or debug the error? Any pointers are much appreciated.
I am using a Datasource to log my audit information per user-request.
The bean is defined in a springContext.xml sourced during initialization.
Whenever the DB is down, the bean initialization fails with following error: DataAccessException.
What I want to do is continue after this exception(since it is ok not to be able to log).
How do I handle this scenario?
Error 500--Internal Server Error
Error creating bean with name 'AuditLogSequenceFactory' defined in ServletContext resource [/WEB-INF/AuditTrailContext.xml]: Instantiation of bean failed; nested exception is org.springframework.beans.factory.BeanDefinitionStoreException: Factory method [public java.lang.String org.springframework.jdbc.support.incrementer.AbstractDataFieldMaxValueIncrementer.nextStringValue() throws org.springframework.dao.DataAccessException] threw exception; nested exception is org.springframework.jdbc.CannotGetJdbcConnectionException: Could not get JDBC Connection; nested exception is java.sql.SQLException: Internal error: Cannot obtain XAConnection weblogic.common.resourcepool.ResourceDisabledException: Pool Simple/Datasource/Local is disabled, cannot allocate resources to applications.. - action - zip:C:/bea/user_projects/domains/Simple/./servers/AdminServer/tmp/_WL_user/simpleWN/j5vygk/war/WEB-INF/lib/_wl_cls_gen.jar!/struts_global.xml:9:60
at com.opensymphony.xwork2.DefaultActionInvocation.createResult(DefaultActionInvocation.java:199)
....
weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:1880)
at weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1310)
at weblogic.work.ExecuteThread.execute(ExecuteThread.java:207)
at weblogic.work.ExecuteThread.run(ExecuteThread.java:179)
Caused by: org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'AuditLogSequenceFactory' defined in ServletContext resource [/WEB-INF/AuditTrailContext.xml]: Instantiation of bean failed; nested exception is org.springframework.beans.factory.BeanDefinitionStoreException: Factory method [public java.lang.String org.springframework.jdbc.support.incrementer.AbstractDataFieldMaxValueIncrementer.nextStringValue() throws org.springframework.dao.DataAccessException] threw exception; nested exception is org.springframework.jdbc.CannotGetJdbcConnectionException: Could not get JDBC Connection; nested exception is java.sql.SQLException: Internal error: Cannot obtain XAConnection weblogic.common.resourcepool.ResourceDisabledException: Pool Simple/Datasource/Local is disabled, cannot allocate resources to applications..
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:442)
....
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:42)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3151)
Caused by: org.springframework.beans.factory.BeanDefinitionStoreException: Factory method [public java.lang.String org.springframework.jdbc.support.incrementer.AbstractDataFieldMaxValueIncrementer.nextStringValue() throws org.springframework.dao.DataAccessException] threw exception; nested exception is org.springframework.jdbc.CannotGetJdbcConnectionException: Could not get JDBC Connection; nested exception is java.sql.SQLException: Internal error: Cannot obtain XAConnection weblogic.common.resourcepool.ResourceDisabledException: Pool Simple/Datasource/Local is disabled, cannot allocate resources to applications..
at org.springframework.beans.factory.support.SimpleInstantiationStrategy.instantiate(SimpleInstantiationStrategy.java:125)
....
at org.apache.struts2.dispatcher.Dispatcher.serviceAction(Dispatcher.java:504)
at org.apache.struts2.dispatcher.FilterDispatcher.doFilter(FilterDispatcher.java:419)
Caused by: org.springframework.jdbc.CannotGetJdbcConnectionException: Could not get JDBC Connection; nested exception is java.sql.SQLException: Internal error: Cannot obtain XAConnection weblogic.common.resourcepool.ResourceDisabledException: Pool Simple/Datasource/Local is disabled, cannot allocate resources to applications..
at org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:80)
.... at com.opensymphony.xwork2.interceptor.ParametersInterceptor.doIntercept(ParametersInterceptor.java:167)
Caused by: java.sql.SQLException: Internal error: Cannot obtain XAConnection weblogic.common.resourcepool.ResourceDisabledException: Pool Simple/Datasource/Local is disabled, cannot allocate resources to applications..
at weblogic.jdbc.jta.DataSource.refreshXAConnAndEnlist(DataSource.java:1295)
at weblogic.jdbc.jta.DataSource.getConnection(DataSource.java:436)
at weblogic.jdbc.jta.DataSource.connect(DataSource.java:392)
at weblogic.jdbc.common.internal.RmiDataSource.getConnection(RmiDataSource.java:359)
.... at org.springframework.beans.factory.support.ConstructorResolver.autowireConstructor(ConstructorResolver.java:126)
If you haven't already, need to make the auditing bean lazy-init to prevent it from actually trying to call the database during initialization.
In your Filter code, put try/catch around the auditing call so you can call the next Filter in the chain.
If this doesn't help perhaps a sample of your Spring configuration and how you invoke your auditing bean would help.