Jenkins failure while installing 'slack Notification' Plugin - jenkins-plugins

Getting below failure message while trying to install 'slack Notification' Plugin in Jenkins. Here Jenkins is running on AWS ec2 instance and I'm trying with 'install without restart' option.
I checked by logging manually to the ec2 instance and can see that there are few hidden files at path "/var/lib/jenkins/plugins/slack/WEB-INF/lib".
jenkins.util.io.CompositeIOException: Unable to delete '/var/lib/jenkins/plugins/slack'. Tried 3 times (of a maximum of 3) waiting 0.1 sec between attempts.
at jenkins.util.io.PathRemover.forceRemoveRecursive(PathRemover.java:99)
at hudson.Util.deleteRecursive(Util.java:293)
at hudson.Util.deleteRecursive(Util.java:282)
at hudson.ClassicPluginStrategy.explode(ClassicPluginStrategy.java:476)
at hudson.ClassicPluginStrategy.createPluginWrapper(ClassicPluginStrategy.java:175)
at hudson.PluginManager.dynamicLoad(PluginManager.java:910)
at hudson.model.UpdateCenter$InstallationJob._run(UpdateCenter.java:2052)
at hudson.model.UpdateCenter$DownloadJob.run(UpdateCenter.java:1740)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at hudson.remoting.AtmostOneThreadExecutor$Worker.run(AtmostOneThreadExecutor.java:111)
at java.lang.Thread.run(Thread.java:748)
jenkins.util.io.CompositeIOException: Unable to remove file /var/lib/jenkins/plugins/slack/WEB-INF/lib/.nfs6cd47c1b7f5964f000013284
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:252)
at jenkins.util.io.PathRemover.tryRemoveFile(PathRemover.java:205)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:216)
at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:226)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:215)
at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:226)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:215)
at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:226)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:215)
at jenkins.util.io.PathRemover.forceRemoveRecursive(PathRemover.java:96)
at hudson.Util.deleteRecursive(Util.java:293)
at hudson.Util.deleteRecursive(Util.java:282)
at hudson.ClassicPluginStrategy.explode(ClassicPluginStrategy.java:476)
at hudson.ClassicPluginStrategy.createPluginWrapper(ClassicPluginStrategy.java:175)
at hudson.PluginManager.dynamicLoad(PluginManager.java:910)
at hudson.model.UpdateCenter$InstallationJob._run(UpdateCenter.java:2052)
at hudson.model.UpdateCenter$DownloadJob.run(UpdateCenter.java:1740)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at hudson.remoting.AtmostOneThreadExecutor$Worker.run(AtmostOneThreadExecutor.java:111)
at java.lang.Thread.run(Thread.java:748)
java.nio.file.FileSystemException: /var/lib/jenkins/plugins/slack/WEB-INF/lib/.nfs6cd47c1b7f5964f000013284: Device or resource busy
at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:244)
at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108)
at java.nio.file.Files.deleteIfExists(Files.java:1165)
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:237)
at jenkins.util.io.PathRemover.tryRemoveFile(PathRemover.java:205)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:216)
at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:226)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:215)
at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:226)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:215)
at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:226)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:215)
at jenkins.util.io.PathRemover.forceRemoveRecursive(PathRemover.java:96)
at hudson.Util.deleteRecursive(Util.java:293)
at hudson.Util.deleteRecursive(Util.java:282)
at hudson.ClassicPluginStrategy.explode(ClassicPluginStrategy.java:476)
at hudson.ClassicPluginStrategy.createPluginWrapper(ClassicPluginStrategy.java:175)
at hudson.PluginManager.dynamicLoad(PluginManager.java:910)
at hudson.model.UpdateCenter$InstallationJob._run(UpdateCenter.java:2052)
at hudson.model.UpdateCenter$DownloadJob.run(UpdateCenter.java:1740)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at hudson.remoting.AtmostOneThreadExecutor$Worker.run(AtmostOneThreadExecutor.java:111)
at java.lang.Thread.run(Thread.java:748)
java.nio.file.FileSystemException: /var/lib/jenkins/plugins/slack/WEB-INF/lib/.nfs6cd47c1b7f5964f000013284: Device or resource busy
at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91)
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102)
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107)
at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:244)
at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108)
at java.nio.file.Files.deleteIfExists(Files.java:1165)
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:241)
at jenkins.util.io.PathRemover.tryRemoveFile(PathRemover.java:205)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:216)
at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:226)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:215)
at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:226)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:215)
at jenkins.util.io.PathRemover.tryRemoveDirectoryContents(PathRemover.java:226)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:215)
at jenkins.util.io.PathRemover.forceRemoveRecursive(PathRemover.java:96)
at hudson.Util.deleteRecursive(Util.java:293)
at hudson.Util.deleteRecursive(Util.java:282)
at hudson.ClassicPluginStrategy.explode(ClassicPluginStrategy.java:476)
at hudson.ClassicPluginStrategy.createPluginWrapper(ClassicPluginStrategy.java:175)
at hudson.PluginManager.dynamicLoad(PluginManager.java:910)
at hudson.model.UpdateCenter$InstallationJob._run(UpdateCenter.java:2052)
at hudson.model.UpdateCenter$DownloadJob.run(UpdateCenter.java:1740)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at hudson.remoting.AtmostOneThreadExecutor$Worker.run(AtmostOneThreadExecutor.java:111)
at java.lang.Thread.run(Thread.java:748)
jenkins.util.io.CompositeIOException: Unable to remove directory /var/lib/jenkins/plugins/slack with directory contents: [/var/lib/jenkins/plugins/slack/WEB-INF]
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:250)
at jenkins.util.io.PathRemover.tryRemoveFile(PathRemover.java:205)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:216)
at jenkins.util.io.PathRemover.forceRemoveRecursive(PathRemover.java:96)
at hudson.Util.deleteRecursive(Util.java:293)
at hudson.Util.deleteRecursive(Util.java:282)
at hudson.ClassicPluginStrategy.explode(ClassicPluginStrategy.java:476)
at hudson.ClassicPluginStrategy.createPluginWrapper(ClassicPluginStrategy.java:175)
at hudson.PluginManager.dynamicLoad(PluginManager.java:910)
at hudson.model.UpdateCenter$InstallationJob._run(UpdateCenter.java:2052)
at hudson.model.UpdateCenter$DownloadJob.run(UpdateCenter.java:1740)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at hudson.remoting.AtmostOneThreadExecutor$Worker.run(AtmostOneThreadExecutor.java:111)
at java.lang.Thread.run(Thread.java:748)
java.nio.file.DirectoryNotEmptyException: /var/lib/jenkins/plugins/slack
at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:242)
at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108)
at java.nio.file.Files.deleteIfExists(Files.java:1165)
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:237)
at jenkins.util.io.PathRemover.tryRemoveFile(PathRemover.java:205)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:216)
at jenkins.util.io.PathRemover.forceRemoveRecursive(PathRemover.java:96)
at hudson.Util.deleteRecursive(Util.java:293)
at hudson.Util.deleteRecursive(Util.java:282)
at hudson.ClassicPluginStrategy.explode(ClassicPluginStrategy.java:476)
at hudson.ClassicPluginStrategy.createPluginWrapper(ClassicPluginStrategy.java:175)
at hudson.PluginManager.dynamicLoad(PluginManager.java:910)
at hudson.model.UpdateCenter$InstallationJob._run(UpdateCenter.java:2052)
at hudson.model.UpdateCenter$DownloadJob.run(UpdateCenter.java:1740)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at hudson.remoting.AtmostOneThreadExecutor$Worker.run(AtmostOneThreadExecutor.java:111)
at java.lang.Thread.run(Thread.java:748)
java.nio.file.DirectoryNotEmptyException: /var/lib/jenkins/plugins/slack
at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:242)
at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108)
at java.nio.file.Files.deleteIfExists(Files.java:1165)
at jenkins.util.io.PathRemover.removeOrMakeRemovableThenRemove(PathRemover.java:241)
at jenkins.util.io.PathRemover.tryRemoveFile(PathRemover.java:205)
at jenkins.util.io.PathRemover.tryRemoveRecursive(PathRemover.java:216)
at jenkins.util.io.PathRemover.forceRemoveRecursive(PathRemover.java:96)
at hudson.Util.deleteRecursive(Util.java:293)
at hudson.Util.deleteRecursive(Util.java:282)
at hudson.ClassicPluginStrategy.explode(ClassicPluginStrategy.java:476)
at hudson.ClassicPluginStrategy.createPluginWrapper(ClassicPluginStrategy.java:175)
at hudson.PluginManager.dynamicLoad(PluginManager.java:910)
at hudson.model.UpdateCenter$InstallationJob._run(UpdateCenter.java:2052)
at hudson.model.UpdateCenter$DownloadJob.run(UpdateCenter.java:1740)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at hudson.remoting.AtmostOneThreadExecutor$Worker.run(AtmostOneThreadExecutor.java:111)
at java.lang.Thread.run(Thread.java:748)
Caused: java.io.IOException: Failed to dynamically deploy this plugin
at hudson.model.UpdateCenter$InstallationJob._run(UpdateCenter.java:2056)
at hudson.model.UpdateCenter$DownloadJob.run(UpdateCenter.java:1740)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at hudson.remoting.AtmostOneThreadExecutor$Worker.run(AtmostOneThreadExecutor.java:111)
at java.lang.Thread.run(Thread.java:748)

Related

Microsoft SQL Server hosted on Windows not connecting on mac on same network through ip

I have deployed Microsoft SQL Server on a Windows machine and tomcat on mac, I want to connect database hosted on Windows machine through IP on same network but getting the following error
Type Exception Report
Message Servlet.init() for servlet [appServlet] threw exception
Description The server encountered an unexpected condition that prevented it from fulfilling the request.
Exception
javax.servlet.ServletException: Servlet.init() for servlet [appServlet] threw exception
org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:542)
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:687)
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:357)
org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:374)
org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:893)
org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1707)
org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
java.lang.Thread.run(Thread.java:748)
Root Cause
org.springframework.transaction.CannotCreateTransactionException: Could not open JPA EntityManager for transaction; nested exception is org.hibernate.exception.JDBCConnectionException: Unable to acquire JDBC Connection
org.springframework.orm.jpa.JpaTransactionManager.doBegin(JpaTransactionManager.java:431)
org.springframework.transaction.support.AbstractPlatformTransactionManager.getTransaction(AbstractPlatformTransactionManager.java:373)
org.springframework.transaction.interceptor.TransactionAspectSupport.createTransactionIfNecessary(TransactionAspectSupport.java:447)
org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:277)
org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:96)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
org.springframework.dao.support.PersistenceExceptionTranslationInterceptor.invoke(PersistenceExceptionTranslationInterceptor.java:136)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
org.springframework.data.jpa.repository.support.CrudMethodMetadataPostProcessor$CrudMethodMetadataPopulatingMethodIntercceptor.invoke(CrudMethodMetadataPostProcessor.java:122)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:213)
com.sun.proxy.$Proxy86.findAll(Unknown Source)
com.avanza.ambit.utils.InitComponents.contextRefreshedEvent(InitComponents.java:27)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
org.springframework.context.event.ApplicationListenerMethodAdapter.doInvoke(ApplicationListenerMethodAdapter.java:256)
org.springframework.context.event.ApplicationListenerMethodAdapter.processEvent(ApplicationListenerMethodAdapter.java:177)
org.springframework.context.event.ApplicationListenerMethodAdapter.onApplicationEvent(ApplicationListenerMethodAdapter.java:140)
org.springframework.context.event.SimpleApplicationEventMulticaster.invokeListener(SimpleApplicationEventMulticaster.java:167)
org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:139)
org.springframework.context.support.AbstractApplicationContext.publishEvent(AbstractApplicationContext.java:393)
org.springframework.context.support.AbstractApplicationContext.publishEvent(AbstractApplicationContext.java:347)
org.springframework.context.support.AbstractApplicationContext.finishRefresh(AbstractApplicationContext.java:883)
org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:546)
org.springframework.web.servlet.FrameworkServlet.configureAndRefreshWebApplicationContext(FrameworkServlet.java:668)
org.springframework.web.servlet.FrameworkServlet.createWebApplicationContext(FrameworkServlet.java:634)
org.springframework.web.servlet.FrameworkServlet.createWebApplicationContext(FrameworkServlet.java:682)
org.springframework.web.servlet.FrameworkServlet.initWebApplicationContext(FrameworkServlet.java:553)
org.springframework.web.servlet.FrameworkServlet.initServletBean(FrameworkServlet.java:494)
org.springframework.web.servlet.HttpServletBean.init(HttpServletBean.java:171)
javax.servlet.GenericServlet.init(GenericServlet.java:158)
org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:542)
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:687)
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:357)
org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:374)
org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:893)
org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1707)
org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
java.lang.Thread.run(Thread.java:748)
Root Cause
org.hibernate.exception.JDBCConnectionException: Unable to acquire JDBC Connection
org.hibernate.exception.internal.SQLStateConversionDelegate.convert(SQLStateConversionDelegate.java:115)
org.hibernate.exception.internal.StandardSQLExceptionConverter.convert(StandardSQLExceptionConverter.java:42)
org.hibernate.engine.jdbc.spi.SqlExceptionHelper.convert(SqlExceptionHelper.java:111)
org.hibernate.engine.jdbc.spi.SqlExceptionHelper.convert(SqlExceptionHelper.java:97)
org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.acquireConnectionIfNeeded(LogicalConnectionManagedImpl.java:118)
org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.getPhysicalConnection(LogicalConnectionManagedImpl.java:145)
org.hibernate.internal.SessionImpl.connection(SessionImpl.java:523)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
org.springframework.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:216)
org.springframework.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:201)
org.springframework.orm.jpa.vendor.HibernateJpaDialect$HibernateConnectionHandle.doGetConnection(HibernateJpaDialect.java:414)
org.springframework.orm.jpa.vendor.HibernateJpaDialect.beginTransaction(HibernateJpaDialect.java:177)
org.springframework.orm.jpa.JpaTransactionManager.doBegin(JpaTransactionManager.java:380)
org.springframework.transaction.support.AbstractPlatformTransactionManager.getTransaction(AbstractPlatformTransactionManager.java:373)
org.springframework.transaction.interceptor.TransactionAspectSupport.createTransactionIfNecessary(TransactionAspectSupport.java:447)
org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:277)
org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:96)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
org.springframework.dao.support.PersistenceExceptionTranslationInterceptor.invoke(PersistenceExceptionTranslationInterceptor.java:136)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
org.springframework.data.jpa.repository.support.CrudMethodMetadataPostProcessor$CrudMethodMetadataPopulatingMethodIntercceptor.invoke(CrudMethodMetadataPostProcessor.java:122)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:213)
com.sun.proxy.$Proxy86.findAll(Unknown Source)
com.avanza.ambit.utils.InitComponents.contextRefreshedEvent(InitComponents.java:27)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
org.springframework.context.event.ApplicationListenerMethodAdapter.doInvoke(ApplicationListenerMethodAdapter.java:256)
org.springframework.context.event.ApplicationListenerMethodAdapter.processEvent(ApplicationListenerMethodAdapter.java:177)
org.springframework.context.event.ApplicationListenerMethodAdapter.onApplicationEvent(ApplicationListenerMethodAdapter.java:140)
org.springframework.context.event.SimpleApplicationEventMulticaster.invokeListener(SimpleApplicationEventMulticaster.java:167)
org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:139)
org.springframework.context.support.AbstractApplicationContext.publishEvent(AbstractApplicationContext.java:393)
org.springframework.context.support.AbstractApplicationContext.publishEvent(AbstractApplicationContext.java:347)
org.springframework.context.support.AbstractApplicationContext.finishRefresh(AbstractApplicationContext.java:883)
org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:546)
org.springframework.web.servlet.FrameworkServlet.configureAndRefreshWebApplicationContext(FrameworkServlet.java:668)
org.springframework.web.servlet.FrameworkServlet.createWebApplicationContext(FrameworkServlet.java:634)
org.springframework.web.servlet.FrameworkServlet.createWebApplicationContext(FrameworkServlet.java:682)
org.springframework.web.servlet.FrameworkServlet.initWebApplicationContext(FrameworkServlet.java:553)
org.springframework.web.servlet.FrameworkServlet.initServletBean(FrameworkServlet.java:494)
org.springframework.web.servlet.HttpServletBean.init(HttpServletBean.java:171)
javax.servlet.GenericServlet.init(GenericServlet.java:158)
org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:542)
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:687)
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:357)
org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:374)
org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:893)
org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1707)
org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
java.lang.Thread.run(Thread.java:748)
Root Cause
com.microsoft.sqlserver.jdbc.SQLServerException: The TCP/IP connection to the host 192.168.0.106, port 1433 has failed. Error: "connect timed out. Verify the connection properties. Make sure that an instance of SQL Server is running on the host and accepting TCP/IP connections at the port. Make sure that TCP connections to the port are not blocked by a firewall.".
com.microsoft.sqlserver.jdbc.SQLServerException.makeFromDriverError(SQLServerException.java:190)
com.microsoft.sqlserver.jdbc.SQLServerException.ConvertConnectExceptionToSQLServerException(SQLServerException.java:241)
com.microsoft.sqlserver.jdbc.SocketFinder.findSocket(IOBuffer.java:2243)
com.microsoft.sqlserver.jdbc.TDSChannel.open(IOBuffer.java:491)
com.microsoft.sqlserver.jdbc.SQLServerConnection.connectHelper(SQLServerConnection.java:1309)
com.microsoft.sqlserver.jdbc.SQLServerConnection.login(SQLServerConnection.java:991)
com.microsoft.sqlserver.jdbc.SQLServerConnection.connect(SQLServerConnection.java:827)
com.microsoft.sqlserver.jdbc.SQLServerDriver.connect(SQLServerDriver.java:1012)
java.sql.DriverManager.getConnection(DriverManager.java:664)
java.sql.DriverManager.getConnection(DriverManager.java:208)
org.springframework.jdbc.datasource.DriverManagerDataSource.getConnectionFromDriverManager(DriverManagerDataSource.java:153)
org.springframework.jdbc.datasource.DriverManagerDataSource.getConnectionFromDriver(DriverManagerDataSource.java:144)
org.springframework.jdbc.datasource.AbstractDriverBasedDataSource.getConnectionFromDriver(AbstractDriverBasedDataSource.java:196)
org.springframework.jdbc.datasource.AbstractDriverBasedDataSource.getConnection(AbstractDriverBasedDataSource.java:159)
org.hibernate.engine.jdbc.connections.internal.DatasourceConnectionProviderImpl.getConnection(DatasourceConnectionProviderImpl.java:122)
org.hibernate.internal.NonContextualJdbcConnectionAccess.obtainConnection(NonContextualJdbcConnectionAccess.java:35)
org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.acquireConnectionIfNeeded(LogicalConnectionManagedImpl.java:115)
org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.getPhysicalConnection(LogicalConnectionManagedImpl.java:145)
org.hibernate.internal.SessionImpl.connection(SessionImpl.java:523)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
org.springframework.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:216)
org.springframework.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:201)
org.springframework.orm.jpa.vendor.HibernateJpaDialect$HibernateConnectionHandle.doGetConnection(HibernateJpaDialect.java:414)
org.springframework.orm.jpa.vendor.HibernateJpaDialect.beginTransaction(HibernateJpaDialect.java:177)
org.springframework.orm.jpa.JpaTransactionManager.doBegin(JpaTransactionManager.java:380)
org.springframework.transaction.support.AbstractPlatformTransactionManager.getTransaction(AbstractPlatformTransactionManager.java:373)
org.springframework.transaction.interceptor.TransactionAspectSupport.createTransactionIfNecessary(TransactionAspectSupport.java:447)
org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:277)
org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:96)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
org.springframework.dao.support.PersistenceExceptionTranslationInterceptor.invoke(PersistenceExceptionTranslationInterceptor.java:136)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
org.springframework.data.jpa.repository.support.CrudMethodMetadataPostProcessor$CrudMethodMetadataPopulatingMethodIntercceptor.invoke(CrudMethodMetadataPostProcessor.java:122)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92)
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:213)
com.sun.proxy.$Proxy86.findAll(Unknown Source)
com.avanza.ambit.utils.InitComponents.contextRefreshedEvent(InitComponents.java:27)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
org.springframework.context.event.ApplicationListenerMethodAdapter.doInvoke(ApplicationListenerMethodAdapter.java:256)
org.springframework.context.event.ApplicationListenerMethodAdapter.processEvent(ApplicationListenerMethodAdapter.java:177)
org.springframework.context.event.ApplicationListenerMethodAdapter.onApplicationEvent(ApplicationListenerMethodAdapter.java:140)
org.springframework.context.event.SimpleApplicationEventMulticaster.invokeListener(SimpleApplicationEventMulticaster.java:167)
org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:139)
org.springframework.context.support.AbstractApplicationContext.publishEvent(AbstractApplicationContext.java:393)
org.springframework.context.support.AbstractApplicationContext.publishEvent(AbstractApplicationContext.java:347)
org.springframework.context.support.AbstractApplicationContext.finishRefresh(AbstractApplicationContext.java:883)
org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:546)
org.springframework.web.servlet.FrameworkServlet.configureAndRefreshWebApplicationContext(FrameworkServlet.java:668)
org.springframework.web.servlet.FrameworkServlet.createWebApplicationContext(FrameworkServlet.java:634)
org.springframework.web.servlet.FrameworkServlet.createWebApplicationContext(FrameworkServlet.java:682)
org.springframework.web.servlet.FrameworkServlet.initWebApplicationContext(FrameworkServlet.java:553)
org.springframework.web.servlet.FrameworkServlet.initServletBean(FrameworkServlet.java:494)
org.springframework.web.servlet.HttpServletBean.init(HttpServletBean.java:171)
javax.servlet.GenericServlet.init(GenericServlet.java:158)
org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:542)
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:687)
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:357)
org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:374)
org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:893)
org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1707)
org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
java.lang.Thread.run(Thread.java:748)
Note The full stack trace of the root cause is available in the server logs.

Flink Lambda serialization error logged when running in standalone server

Version: Flink 1.12, java 1.11.
No issues while runnning in local environment. When running in standalone cluster with below config
INFO [] - Loading configuration property: jobmanager.rpc.address, localhost
INFO [] - Loading configuration property: jobmanager.rpc.port, 6123
INFO [] - Loading configuration property: taskmanager.cpu.cores, 1.79
INFO [] - Loading configuration property: taskmanager.memory.task.heap.size, 4096m
INFO [] - Loading configuration property: taskmanager.memory.task.off-heap.size, 4096m
INFO [] - Loading configuration property: taskmanager.memory.managed.size, 128m
INFO [] - Loading configuration property: taskmanager.memory.network.min, 64m
WARN [] - Error while trying to split key and value in configuration file /Users/vgamini/tools/flink-1.12.0/conf/flink-conf.yaml:44: "taskmanager.memory.network.max:64m"
INFO [] - Loading configuration property: jobmanager.memory.flink.size, 4096m
INFO [] - Loading configuration property: taskmanager.numberOfTaskSlots, 1
INFO [] - Loading configuration property: parallelism.default, 1
I am seeing the below error in task manager logs - but the job went to running state
2020-12-29 15:16:31,322 WARN org.apache.flink.runtime.taskmanager.Task [] - Source: Custom Source (1/3)#0 (5c850a62dc24ac6ccea8da166d5cc8f6) switched from DEPLOYING to FAILED.
org.apache.flink.streaming.runtime.tasks.StreamTaskException: Could not instantiate outputs in order.
at org.apache.flink.streaming.api.graph.StreamConfig.getOutEdgesInOrder(StreamConfig.java:470) ~[flink-dist_2.12-1.12.0.jar:1.12.0]
at org.apache.flink.streaming.runtime.tasks.StreamTask.createRecordWriters(StreamTask.java:1138) ~[flink-dist_2.12-1.12.0.jar:1.12.0]
at org.apache.flink.streaming.runtime.tasks.StreamTask.createRecordWriterDelegate(StreamTask.java:1122) ~[flink-dist_2.12-1.12.0.jar:1.12.0]
at org.apache.flink.streaming.runtime.tasks.StreamTask.<init>(StreamTask.java:290) ~[flink-dist_2.12-1.12.0.jar:1.12.0]
at org.apache.flink.streaming.runtime.tasks.StreamTask.<init>(StreamTask.java:277) ~[flink-dist_2.12-1.12.0.jar:1.12.0]
at org.apache.flink.streaming.runtime.tasks.SourceStreamTask.<init>(SourceStreamTask.java:73) ~[flink-dist_2.12-1.12.0.jar:1.12.0]
at org.apache.flink.streaming.runtime.tasks.SourceStreamTask.<init>(SourceStreamTask.java:69) ~[flink-dist_2.12-1.12.0.jar:1.12.0]
at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:?]
at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
at java.lang.reflect.Constructor.newInstance(Constructor.java:490) ~[?:?]
at org.apache.flink.runtime.taskmanager.Task.loadAndInstantiateInvokable(Task.java:1373) [flink-dist_2.12-1.12.0.jar:1.12.0]
at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:700) [flink-dist_2.12-1.12.0.jar:1.12.0]
at org.apache.flink.runtime.taskmanager.Task.run(Task.java:547) [flink-dist_2.12-1.12.0.jar:1.12.0]
at java.lang.Thread.run(Thread.java:834) [?:?]
Caused by: java.lang.ClassCastException: cannot assign instance of java.lang.invoke.SerializedLambda to field org.apache.flink.streaming.runtime.partitioner.KeyGroupStreamPartitioner.keySelector of type org.apache.flink.api.java.functions.KeySelector in instance of org.apache.flink.streaming.runtime.partitioner.KeyGroupStreamPartitioner
at java.io.ObjectStreamClass$FieldReflector.setObjFieldValues(ObjectStreamClass.java:2190) ~[?:?]
at java.io.ObjectStreamClass$FieldReflector.checkObjectFieldValueTypes(ObjectStreamClass.java:2153) ~[?:?]
at java.io.ObjectStreamClass.checkObjFieldValueTypes(ObjectStreamClass.java:1407) ~[?:?]
at java.io.ObjectInputStream.defaultCheckFieldValues(ObjectInputStream.java:2371) ~[?:?]
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2278) ~[?:?]
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2087) ~[?:?]
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1594) ~[?:?]
at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2355) ~[?:?]
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2249) ~[?:?]
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2087) ~[?:?]
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1594) ~[?:?]
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:430) ~[?:?]
at java.util.ArrayList.readObject(ArrayList.java:928) ~[?:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:?]
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
at java.lang.reflect.Method.invoke(Method.java:566) ~[?:?]
at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1160) ~[?:?]
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2216) ~[?:?]
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2087) ~[?:?]
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1594) ~[?:?]
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:430) ~[?:?]
at org.apache.flink.util.InstantiationUtil.deserializeObject(InstantiationUtil.java:576) ~[flink-dist_2.12-1.12.0.jar:1.12.0]
at org.apache.flink.util.InstantiationUtil.deserializeObject(InstantiationUtil.java:562) ~[flink-dist_2.12-1.12.0.jar:1.12.0]
at org.apache.flink.util.InstantiationUtil.deserializeObject(InstantiationUtil.java:550) ~[flink-dist_2.12-1.12.0.jar:1.12.0]
at org.apache.flink.util.InstantiationUtil.readObjectFromConfig(InstantiationUtil.java:511) ~[flink-dist_2.12-1.12.0.jar:1.12.0]
at org.apache.flink.streaming.api.graph.StreamConfig.getOutEdgesInOrder(StreamConfig.java:467) ~[flink-dist_2.12-1.12.0.jar:1.12.0]
Is it just a warning? or will there be any impact because of this error.
Usecase:
Take a kafka input stream and key by a field to apply window grouping.
Note: While creating the fat jar - all the recommendations has been followed.
Do anyone faced the same issue?
The execution shows 2 tasks while the logs show tasks as 1/3. Can somebody help understanding it?

This problem occurs when you run Flink in Application Mode, but only your JobManager has the job jar in its classpath (or the usrlib/ folder).
To fix this issue, make the job jar available on your TaskManger instances as well.
You can do this by either building a custom docker image, containing the required jar(s), or by mounting them.
The exception in Flink 1.13.1 looked like this for me:
2021-08-27 11:44:45,788 INFO org.apache.flink.runtime.scheduler.adaptive.AdaptiveScheduler [] - Job 857fbf37156b20a8d445b9e3d9a465ab reached terminal state FAILED.
org.apache.flink.util.SerializedThrowable: Recovery is suppressed by NoRestartBackoffTimeStrategy
at org.apache.flink.runtime.scheduler.adaptive.AdaptiveScheduler.howToHandleFailure(AdaptiveScheduler.java:1073) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.runtime.scheduler.adaptive.Executing.handleAnyFailure(Executing.java:88) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.runtime.scheduler.adaptive.Executing.updateTaskExecutionState(Executing.java:114) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.runtime.scheduler.adaptive.AdaptiveScheduler.lambda$updateTaskExecutionState$4(AdaptiveScheduler.java:471) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.runtime.scheduler.adaptive.State.tryCall(State.java:142) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.runtime.scheduler.adaptive.AdaptiveScheduler.updateTaskExecutionState(AdaptiveScheduler.java:468) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.runtime.scheduler.SchedulerNG.updateTaskExecutionState(SchedulerNG.java:79) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.runtime.jobmaster.JobMaster.updateTaskExecutionState(JobMaster.java:435) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_302]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_302]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_302]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_302]
at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcInvocation(AkkaRpcActor.java:305) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:212) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:77) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:158) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:26) [flink-dist_2.11-1.13.1.jar:1.13.1]
at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:21) [flink-dist_2.11-1.13.1.jar:1.13.1]
at scala.PartialFunction$class.applyOrElse(PartialFunction.scala:123) [flink-dist_2.11-1.13.1.jar:1.13.1]
at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:21) [flink-dist_2.11-1.13.1.jar:1.13.1]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:170) [flink-dist_2.11-1.13.1.jar:1.13.1]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171) [flink-dist_2.11-1.13.1.jar:1.13.1]
at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171) [flink-dist_2.11-1.13.1.jar:1.13.1]
at akka.actor.Actor$class.aroundReceive(Actor.scala:517) [flink-dist_2.11-1.13.1.jar:1.13.1]
at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:225) [flink-dist_2.11-1.13.1.jar:1.13.1]
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:592) [flink-dist_2.11-1.13.1.jar:1.13.1]
at akka.actor.ActorCell.invoke(ActorCell.scala:561) [flink-dist_2.11-1.13.1.jar:1.13.1]
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:258) [flink-dist_2.11-1.13.1.jar:1.13.1]
at akka.dispatch.Mailbox.run(Mailbox.scala:225) [flink-dist_2.11-1.13.1.jar:1.13.1]
at akka.dispatch.Mailbox.exec(Mailbox.scala:235) [flink-dist_2.11-1.13.1.jar:1.13.1]
at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) [flink-dist_2.11-1.13.1.jar:1.13.1]
at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) [flink-dist_2.11-1.13.1.jar:1.13.1]
at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) [flink-dist_2.11-1.13.1.jar:1.13.1]
at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) [flink-dist_2.11-1.13.1.jar:1.13.1]
Caused by: org.apache.flink.util.SerializedThrowable: Could not instantiate outputs in order.
at org.apache.flink.streaming.api.graph.StreamConfig.getOutEdgesInOrder(StreamConfig.java:485) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.streaming.runtime.tasks.StreamTask.createRecordWriters(StreamTask.java:1338) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.streaming.runtime.tasks.StreamTask.createRecordWriterDelegate(StreamTask.java:1322) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.streaming.runtime.tasks.StreamTask.<init>(StreamTask.java:327) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.streaming.runtime.tasks.StreamTask.<init>(StreamTask.java:308) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.streaming.runtime.tasks.SourceStreamTask.<init>(SourceStreamTask.java:76) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.streaming.runtime.tasks.SourceStreamTask.<init>(SourceStreamTask.java:72) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_302]
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_302]
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_302]
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_302]
at org.apache.flink.runtime.taskmanager.Task.loadAndInstantiateInvokable(Task.java:1524) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:730) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.runtime.taskmanager.Task.run(Task.java:566) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_302]
Caused by: org.apache.flink.util.SerializedThrowable: cannot assign instance of java.lang.invoke.SerializedLambda to field org.apache.flink.streaming.runtime.partitioner.KeyGroupStreamPartitioner.keySelector of type org.apache.flink.api.java.functions.KeySelector in instance of org.apache.flink.streaming.runtime.partitioner.KeyGroupStreamPartitioner
at java.io.ObjectStreamClass$FieldReflector.setObjFieldValues(ObjectStreamClass.java:2301) ~[?:1.8.0_302]
at java.io.ObjectStreamClass.setObjFieldValues(ObjectStreamClass.java:1431) ~[?:1.8.0_302]
at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2411) ~[?:1.8.0_302]
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2329) ~[?:1.8.0_302]
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2187) ~[?:1.8.0_302]
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1667) ~[?:1.8.0_302]
at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2405) ~[?:1.8.0_302]
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2329) ~[?:1.8.0_302]
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2187) ~[?:1.8.0_302]
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1667) ~[?:1.8.0_302]
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:503) ~[?:1.8.0_302]
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:461) ~[?:1.8.0_302]
at java.util.ArrayList.readObject(ArrayList.java:799) ~[?:1.8.0_302]
at sun.reflect.GeneratedMethodAccessor5.invoke(Unknown Source) ~[?:?]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_302]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_302]
at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1184) ~[?:1.8.0_302]
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2296) ~[?:1.8.0_302]
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2187) ~[?:1.8.0_302]
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1667) ~[?:1.8.0_302]
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:503) ~[?:1.8.0_302]
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:461) ~[?:1.8.0_302]
at org.apache.flink.util.InstantiationUtil.deserializeObject(InstantiationUtil.java:615) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.util.InstantiationUtil.deserializeObject(InstantiationUtil.java:600) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.util.InstantiationUtil.deserializeObject(InstantiationUtil.java:587) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.util.InstantiationUtil.readObjectFromConfig(InstantiationUtil.java:541) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.streaming.api.graph.StreamConfig.getOutEdgesInOrder(StreamConfig.java:482) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.streaming.runtime.tasks.StreamTask.createRecordWriters(StreamTask.java:1338) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.streaming.runtime.tasks.StreamTask.createRecordWriterDelegate(StreamTask.java:1322) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.streaming.runtime.tasks.StreamTask.<init>(StreamTask.java:327) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.streaming.runtime.tasks.StreamTask.<init>(StreamTask.java:308) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.streaming.runtime.tasks.SourceStreamTask.<init>(SourceStreamTask.java:76) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.streaming.runtime.tasks.SourceStreamTask.<init>(SourceStreamTask.java:72) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_302]
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_302]
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_302]
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_302]
at org.apache.flink.runtime.taskmanager.Task.loadAndInstantiateInvokable(Task.java:1524) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:730) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at org.apache.flink.runtime.taskmanager.Task.run(Task.java:566) ~[flink-dist_2.11-1.13.1.jar:1.13.1]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_302]

What causes error "The timeout expired before connecting to the routing destination."

I need some help. My company is testing our disaster recovery procedure and after switching our PRD environment to our DR environment we are getting this error with our applications. We are using an MSSQL Always on database setup with a spring boot hikari connection pool, if that is a contributing factor .
The TCP/IP connection to the host , port has failed.
Error: "The timeout expired before connecting to the routing
destination.
Questions:
1) What is the cause of this error?
2) What is an easy way of replicating this?
3) Is this anyway related to the a command timeout? Any idea how to set that value in a hikari pool with spring boot?
4) Assuming that applications settings are fine? Any other possible reason this can happen (Network, database setup, DNS, hardware)
LOG
2019-04-06 07:03:45:866 INFO [http-nio-50209-exec-9] ph.com.bbb.tbg.aaaa.ws.aop.ValidateHeadersAspect : Validating channel 3a96b206-d29b-4746-8f90-12048683e5xx
2019-04-06 07:03:45:900 INFO [http-nio-50209-exec-9] ph.com.bbb.tbg.aaaa.ws.controller.InstitutionWebServiceController : Retrieving institutions where CODE= 99999
2019-04-06 07:03:45:900 INFO [http-nio-50209-exec-9] ph.com.bbb.tbg.aaaa.config.ReadOnlyConnectionAspect : Using read-only Database connection.
2019-04-06 07:03:45:902 INFO [http-nio-50209-exec-9] com.zaxxer.hikari.HikariDataSource : WS Readonly Connection Pool - Started.
2019-04-06 07:03:55:862 WARN [WS Master Connection Pool housekeeper] com.zaxxer.hikari.pool.ProxyLeakTask : Connection leak detection triggered for ConnectionID:417 ClientConnectionId: 6649536a-891f-4756-a75b-38fa5b6d7bf9, stack trace follows
java.lang.Exception: Apparent connection leak detected
at net.ttddyy.dsproxy.support.ProxyDataSource.getConnection(ProxyDataSource.java:63)
at com.github.gavlyukovskiy.boot.jdbc.decorator.DataSourceDecoratorInterceptor.invoke(DataSourceDecoratorInterceptor.java:53)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:655)
at com.zaxxer.hikari.HikariDataSource$$EnhancerBySpringCGLIB$$69498630.getConnection(<generated>)
at org.springframework.jdbc.datasource.lookup.AbstractRoutingDataSource.getConnection(AbstractRoutingDataSource.java:164)
at org.hibernate.engine.jdbc.connections.internal.DatasourceConnectionProviderImpl.getConnection(DatasourceConnectionProviderImpl.java:122)
at org.hibernate.internal.AbstractSessionImpl$NonContextualJdbcConnectionAccess.obtainConnection(AbstractSessionImpl.java:386)
at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.acquireConnectionIfNeeded(LogicalConnectionManagedImpl.java:87)
at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.getPhysicalConnection(LogicalConnectionManagedImpl.java:112)
at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.getConnectionForTransactionManagement(LogicalConnectionManagedImpl.java:230)
at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.begin(LogicalConnectionManagedImpl.java:237)
at org.hibernate.resource.transaction.backend.jdbc.internal.JdbcResourceLocalTransactionCoordinatorImpl$TransactionDriverControlImpl.begin(JdbcResourceLocalTransactionCoordinatorImpl.java:214)
at org.hibernate.engine.transaction.internal.TransactionImpl.begin(TransactionImpl.java:52)
at org.hibernate.internal.SessionImpl.beginTransaction(SessionImpl.java:1512)
at org.hibernate.jpa.internal.TransactionImpl.begin(TransactionImpl.java:45)
at org.springframework.orm.jpa.vendor.HibernateJpaDialect.beginTransaction(HibernateJpaDialect.java:189)
at org.springframework.orm.jpa.JpaTransactionManager.doBegin(JpaTransactionManager.java:380)at org.springframework.transaction.support.AbstractPlatformTransactionManager.getTransaction(AbstractPlatformTransactionManager.java:373)
at org.springframework.transaction.interceptor.TransactionAspectSupport.createTransactionIfNecessary(TransactionAspectSupport.java:430)
at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:276)
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:96)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:655)
at ph.com.bbb.tbg.aaaa.ws.controller.InstitutionWebServiceController$$EnhancerBySpringCGLIB$$fe88c7fb.searchInstitutions(<generated>)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:221)
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:136)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:114)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:827)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:738)
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:963)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:861)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:622)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:729)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:292)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:197)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
at org.springframework.boot.web.support.ErrorPageFilter.doFilter(ErrorPageFilter.java:119)
at org.springframework.boot.web.support.ErrorPageFilter.access$000(ErrorPageFilter.java:61)
at org.springframework.boot.web.support.ErrorPageFilter$1.doFilterInternal(ErrorPageFilter.java:94)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.springframework.boot.web.support.ErrorPageFilter.doFilter(ErrorPageFilter.java:112)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:212)at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:106)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:502)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:141)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:79)
at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:616)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:88)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:528)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1099)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:670)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1520)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1476)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:745)
2019-04-06 07:04:03:908 ERROR [http-nio-50209-exec-9] ph.com.bbb.tbg.aaaa.ws.controller.ExceptionHandlerController : Could not open JPA EntityManager for transaction; nested exception is com.zaxxer.hikari.pool.HikariPool$PoolInitializationException: Failed to initialize pool: The TCP/IP connection to the host db01.aaaa.bbb.com.ph, port 1433 has failed. Error: "The timeout expired before connecting to the routing destination.". ClientConnectionId:47ecb453-1bec-44d3-8a58-2ad25e9fdd2f org.springframework.transaction.CannotCreateTransactionException: Could not open JPA EntityManager for transaction; nested exception is com.zaxxer.hikari.pool.HikariPool$PoolInitializationException: Failed to initialize pool: The TCP/IP connection to the host db01.aaaa.bbb.com.ph, port 1433 has failed. Error: "The timeout expired before connecting to the routing destination.". ClientConnectionId:47ecb453-1bec-44d3-8a58-2ad25e9fdd2f
at org.springframework.orm.jpa.JpaTransactionManager.doBegin(JpaTransactionManager.java:431)
at org.springframework.transaction.support.AbstractPlatformTransactionManager.handleExistingTransaction(AbstractPlatformTransactionManager.java:429)
at org.springframework.transaction.support.AbstractPlatformTransactionManager.getTransaction(AbstractPlatformTransactionManager.java:349)
at org.springframework.transaction.support.TransactionTemplate.execute(TransactionTemplate.java:130)
at ph.com.bbb.tbg.aaaa.config.ReadOnlyConnectionAspect.around(ReadOnlyConnectionAspect.java:52)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:629)
at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:618)
at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:70)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:168)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:655)
at ph.com.bbb.tbg.aaaa.ws.service.InstitutionServiceImpl$$EnhancerBySpringCGLIB$$5b01332b.searchInstitutions(<generated>)
at ph.com.bbb.tbg.aaaa.ws.controller.InstitutionWebServiceController.searchInstitutions(InstitutionWebServiceController.java:172)
at ph.com.bbb.tbg.aaaa.ws.controller.InstitutionWebServiceController$$FastClassBySpringCGLIB$$8223adbe.invoke(<generated>)
at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:720)at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
at org.springframework.aop.framework.adapter.MethodBeforeAdviceInterceptor.invoke(MethodBeforeAdviceInterceptor.java:52)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:168)
at org.springframework.transaction.interceptor.TransactionInterceptor$1.proceedWithInvocation(TransactionInterceptor.java:99)
at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:281)
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:96)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:92)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:655)
at ph.com.bbb.tbg.aaaa.ws.controller.InstitutionWebServiceController$$EnhancerBySpringCGLIB$$fe88c7fb.searchInstitutions(<generated>)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:221)
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:136)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:114)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:827)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:738)
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:963)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:861)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:622)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:729)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:292)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:197)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
at org.springframework.boot.web.support.ErrorPageFilter.doFilter(ErrorPageFilter.java:119)
at org.springframework.boot.web.support.ErrorPageFilter.access$000(ErrorPageFilter.java:61)
at org.springframework.boot.web.support.ErrorPageFilter$1.doFilterInternal(ErrorPageFilter.java:94)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.springframework.boot.web.support.ErrorPageFilter.doFilter(ErrorPageFilter.java:112)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:240)at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:207)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:212)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:106)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:502)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:141)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:79)
at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:616)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:88)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:528)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1099)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:670)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1520)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1476)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:745)
Caused by: com.zaxxer.hikari.pool.HikariPool$PoolInitializationException: Failed to initialize pool: The TCP/IP connection to the host db01.aaaa.bbb.com.ph, port 1433 has failed. Error: "The timeout expired before connecting to the routing destination.". ClientConnectionId:47ecb453-1bec-44d3-8a58-2ad25e9fdd2f
at com.zaxxer.hikari.pool.HikariPool.checkFailFast(HikariPool.java:516)
at com.zaxxer.hikari.pool.HikariPool.<init>(HikariPool.java:118)
at com.zaxxer.hikari.HikariDataSource.getConnection(HikariDataSource.java:94)
at net.ttddyy.dsproxy.support.ProxyDataSource.getConnection(ProxyDataSource.java:63)
at com.github.gavlyukovskiy.boot.jdbc.decorator.DataSourceDecoratorInterceptor.invoke(DataSourceDecoratorInterceptor.java:53)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:655)
at com.zaxxer.hikari.HikariDataSource$$EnhancerBySpringCGLIB$$69498630.getConnection(<generated>)
at org.springframework.jdbc.datasource.lookup.AbstractRoutingDataSource.getConnection(AbstractRoutingDataSource.java:164)
at org.hibernate.engine.jdbc.connections.internal.DatasourceConnectionProviderImpl.getConnection(DatasourceConnectionProviderImpl.java:122)
at org.hibernate.internal.AbstractSessionImpl$NonContextualJdbcConnectionAccess.obtainConnection(AbstractSessionImpl.java:386)
at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.acquireConnectionIfNeeded(LogicalConnectionManagedImpl.java:87)
at org.hibernate.resource.jdbc.internal.LogicalConnectionManagedImpl.getPhysicalConnection(LogicalConnectionManagedImpl.java:112)
at org.hibernate.internal.SessionImpl.connection(SessionImpl.java:489)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.springframework.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:216)
at org.springframework.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:201)
at org.springframework.orm.jpa.vendor.HibernateJpaDialect$HibernateConnectionHandle.doGetConnection(HibernateJpaDialect.java:414)
at org.springframework.orm.jpa.vendor.HibernateJpaDialect.beginTransaction(HibernateJpaDialect.java:177)
at org.springframework.orm.jpa.JpaTransactionManager.doBegin(JpaTransactionManager.java:380)... 80 more
Caused by: com.microsoft.sqlserver.jdbc.SQLServerException: The TCP/IP connection to the host db01.aaaa.bbb.com.ph, port 1433 has failed. Error: "The timeout expired before connecting to the routing destination.". ClientConnectionId:47ecb453-1bec-44d3-8a58-2ad25e9fdd2f
at com.microsoft.sqlserver.jdbc.SQLServerConnection.terminate(SQLServerConnection.java:2670)
at com.microsoft.sqlserver.jdbc.SQLServerConnection.terminate(SQLServerConnection.java:2658)
at com.microsoft.sqlserver.jdbc.SQLServerConnection.login(SQLServerConnection.java:1976)
at com.microsoft.sqlserver.jdbc.SQLServerConnection.connectInternal(SQLServerConnection.java:1762)
at com.microsoft.sqlserver.jdbc.SQLServerConnection.connect(SQLServerConnection.java:1077)
at com.microsoft.sqlserver.jdbc.SQLServerDriver.connect(SQLServerDriver.java:623)
at com.zaxxer.hikari.util.DriverDataSource.getConnection(DriverDataSource.java:95)
at com.zaxxer.hikari.util.DriverDataSource.getConnection(DriverDataSource.java:101)
at com.zaxxer.hikari.pool.PoolBase.newConnection(PoolBase.java:316)
at com.zaxxer.hikari.pool.HikariPool.checkFailFast(HikariPool.java:506)

Flink 1.6 TimeoutException while submitting the job

I am using flink 1.6 and I am getting random timeout issues while submitting the job in the flink Job manager.
Caused by: org.apache.flink.runtime.concurrent.FutureUtils$RetryException: Could not complete the operation. Exception is not retryable.
at org.apache.flink.runtime.concurrent.FutureUtils.lambda$retryOperationWithDelay$5(FutureUtils.java:213)
at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:760)
at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:736)
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474)
at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1977)
at org.apache.flink.runtime.concurrent.FutureUtils$Timeout.run(FutureUtils.java:793)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.CompletionException: java.util.concurrent.TimeoutException
at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292)
at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308)
at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:593)
at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:577)
... 10 more
Caused by: java.util.concurrent.TimeoutException

Apache Zeppelin error local jar not exist

java.lang.RuntimeException: Warning: Local jar
C:\Zeppelin\zeppelin-0.8.0-bin-all\bin\54480 does not exist, skipping.
Warning: Local jar C:\Zeppelin\zeppelin-0.8.0-bin-all\bin\10.10.10.122
does not exist, skipping. java.lang.ClassNotFoundException:
org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer at
java.net.URLClassLoader.findClass(URLClassLoader.java:381) at
java.lang.ClassLoader.loadClass(ClassLoader.java:424) at
java.lang.ClassLoader.loadClass(ClassLoader.java:357) at
java.lang.Class.forName0(Native Method) at
java.lang.Class.forName(Class.java:348) at
org.apache.spark.util.Utils$.classForName(Utils.scala:235) at
org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:836)
at
org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:197)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:227)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:136) at
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 2018-09-18
10:33:29 INFO ShutdownHookManager:54 - Shutdown hook called 2018-09-18
10:33:29 INFO ShutdownHookManager:54 - Deleting directory
C:\Users\Polichetti\AppData\Local\Temp\spark-e1cca18a-e05a-4539-b1b6-2f56a8ab27aa
at
org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess.start(RemoteInterpreterManagedProcess.java:205)
at
org.apache.zeppelin.interpreter.ManagedInterpreterGroup.getOrCreateInterpreterProcess(ManagedInterpreterGroup.java:64)
at
org.apache.zeppelin.interpreter.remote.RemoteInterpreter.getOrCreateInterpreterProcess(RemoteInterpreter.java:111)
at
org.apache.zeppelin.interpreter.remote.RemoteInterpreter.internal_create(RemoteInterpreter.java:164)
at
org.apache.zeppelin.interpreter.remote.RemoteInterpreter.open(RemoteInterpreter.java:132)
at
org.apache.zeppelin.interpreter.remote.RemoteInterpreter.getFormType(RemoteInterpreter.java:299)
at org.apache.zeppelin.notebook.Paragraph.jobRun(Paragraph.java:407)
at org.apache.zeppelin.scheduler.Job.run(Job.java:188) at
org.apache.zeppelin.scheduler.RemoteScheduler$JobRunner.run(RemoteScheduler.java:307)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266) at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)

Please James, this is not the way that Stackoverflow works... Do some research If you have an error log, and don't throw to here like trash.

Resources