Whenever DB is restart during maintenance we see CPU spike in our application , and below thread are using 99% of the CPU and issue is only resolved when application is restarted ...
We have tried all the timeout and jdbc settings and it does not help .. what setting we can used so that this thread are released and does not cause the CPU spike
Stack tread
"Tomcat JDBC Pool Cleaner[622488023:1685120723181]" #22 daemon prio=5 os_prio=0 cpu=38.25ms elapsed=1398.92s tid=0x00007f61cd2ba1e0 nid=0x98 waiting for monitor entry [0x00007f59682d8000]
java.lang.Thread.State: BLOCKED (on object monitor)
at oracle.jdbc.driver.PhysicalConnection.close(PhysicalConnection.java:1978)
- waiting to lock <0x00000000c17162b0> (a oracle.jdbc.driver.T4CConnection)
at org.apache.tomcat.jdbc.pool.PooledConnection.disconnect(PooledConnection.java:414)
at org.apache.tomcat.jdbc.pool.PooledConnection.release(PooledConnection.java:650)
at org.apache.tomcat.jdbc.pool.ConnectionPool.release(ConnectionPool.java:635)
at org.apache.tomcat.jdbc.pool.ConnectionPool.abandon(ConnectionPool.java:589)
at org.apache.tomcat.jdbc.pool.ConnectionPool.checkAbandoned(ConnectionPool.java:1087)
at org.apache.tomcat.jdbc.pool.ConnectionPool$PoolCleaner.run(ConnectionPool.java:1589)
at java.util.TimerThread.mainLoop([email protected]/Timer.java:566)
at java.util.TimerThread.run([email protected]/Timer.java:516)
Locked ownable synchronizers:
- <0x00000000c17113a0> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync)
"pool-8-thread-1" #52 daemon prio=5 os_prio=0 cpu=956574.70ms elapsed=1383.92s tid=0x00007f61980261c0 nid=0xb8 runnable [0x00007f592b7fd000]
java.lang.Thread.State: RUNNABLE
at oracle.net.ns.NIOPacket.writeToSocketChannel(NIOPacket.java:301)
at oracle.net.ns.NIONSDataChannel.writeDataToSocketChannel(NIONSDataChannel.java:173)
at oracle.net.ns.NIONSDataChannel.writeDataToSocketChannel(NIONSDataChannel.java:124)
at oracle.jdbc.driver.T4CMAREngineNIO.flush(T4CMAREngineNIO.java:727)
at oracle.jdbc.driver.T4CMAREngineNIO.prepareForUnmarshall(T4CMAREngineNIO.java:733)
at oracle.jdbc.driver.T4CMAREngineNIO.unmarshalUB1(T4CMAREngineNIO.java:413)
at oracle.jdbc.driver.T4CTTIfun.receive(T4CTTIfun.java:485)
at oracle.jdbc.driver.T4CTTIfun.doRPC(T4CTTIfun.java:252)
at oracle.jdbc.driver.T4C7Ocommoncall.doOCOMMIT(T4C7Ocommoncall.java:72)
at oracle.jdbc.driver.T4CConnection.doCommit(T4CConnection.java:961)
- locked <0x00000000c17162b0> (a oracle.jdbc.driver.T4CConnection)
at oracle.jdbc.driver.PhysicalConnection.commit(PhysicalConnection.java:1937)
- locked <0x00000000c17162b0> (a oracle.jdbc.driver.T4CConnection)
at oracle.jdbc.driver.PhysicalConnection.commit(PhysicalConnection.java:1942)
at jdk.internal.reflect.GeneratedMethodAccessor74.invoke(Unknown Source)
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke([email protected]/DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke([email protected]/Method.java:568)
at org.apache.tomcat.jdbc.pool.ProxyConnection.invoke(ProxyConnection.java:131)
at org.apache.tomcat.jdbc.pool.JdbcInterceptor.invoke(JdbcInterceptor.java:107)
at org.apache.tomcat.jdbc.pool.interceptor.AbstractCreateStatementInterceptor.invoke(AbstractCreateStatementInterceptor.java:78)
at org.apache.tomcat.jdbc.pool.JdbcInterceptor.invoke(JdbcInterceptor.java:107)
at org.apache.tomcat.jdbc.pool.DisposableConnectionFacade.invoke(DisposableConnectionFacade.java:81)
at jdk.proxy2.$Proxy83.commit(jdk.proxy2/Unknown Source)
at org.hibernate.resource.jdbc.internal.AbstractLogicalConnectionImplementor.commit(AbstractLogicalConnectionImplementor.java:86)
at org.hibernate.resource.transaction.backend.jdbc.internal.JdbcResourceLocalTransactionCoordinatorImpl$TransactionDriverControlImpl.commit(JdbcResourceLocalTransactionCoordinatorImpl.java:282)
at org.hibernate.engine.transaction.internal.TransactionImpl.commit(TransactionImpl.java:101)
at io.dropwizard.hibernate.SessionFactoryHealthCheck.lambda$check$0(SessionFactoryHealthCheck.java:56)
at io.dropwizard.hibernate.SessionFactoryHealthCheck$$Lambda$1617/0x00000008016fe778.call(Unknown Source)
at java.util.concurrent.FutureTask.run([email protected]/FutureTask.java:264)
at java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/ThreadPoolExecutor.java:1136)
at java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/ThreadPoolExecutor.java:635)
at com.codahale.metrics.InstrumentedThreadFactory$InstrumentedRunnable.run(InstrumentedThreadFactory.java:66)
at java.lang.Thread.run([email protected]/Thread.java:833)
Locked ownable synchronizers:
- <0x00000000cdd67ce0> (a java.util.concurrent.ThreadPoolExecutor$Worker)
"pool-9-thread-1" #66 daemon prio=5 os_prio=0 cpu=941351.73ms elapsed=942.41s tid=0x00007f6178006bb0 nid=0x11d runnable [0x00007f59384ed000]
java.lang.Thread.State: RUNNABLE
at oracle.net.ns.NIOPacket.writeToSocketChannel(NIOPacket.java:301)
at oracle.net.ns.NIONSDataChannel.writeDataToSocketChannel(NIONSDataChannel.java:173)
at oracle.net.ns.NIONSDataChannel.writeDataToSocketChannel(NIONSDataChannel.java:124)
at oracle.jdbc.driver.T4CMAREngineNIO.flush(T4CMAREngineNIO.java:727)
at oracle.jdbc.driver.T4CMAREngineNIO.prepareForUnmarshall(T4CMAREngineNIO.java:733)
at oracle.jdbc.driver.T4CMAREngineNIO.unmarshalUB1(T4CMAREngineNIO.java:413)
at oracle.jdbc.driver.T4CTTIfun.receive(T4CTTIfun.java:485)
at oracle.jdbc.driver.T4CTTIfun.doRPC(T4CTTIfun.java:252)
at oracle.jdbc.driver.T4C7Ocommoncall.doOCOMMIT(T4C7Ocommoncall.java:72)
at oracle.jdbc.driver.T4CConnection.doCommit(T4CConnection.java:961)
- locked <0x00000000c19b97c0> (a oracle.jdbc.driver.T4CConnection)
at oracle.jdbc.driver.PhysicalConnection.commit(PhysicalConnection.java:1937)
- locked <0x00000000c19b97c0> (a oracle.jdbc.driver.T4CConnection)
at oracle.jdbc.driver.PhysicalConnection.commit(PhysicalConnection.java:1942)
at jdk.internal.reflect.GeneratedMethodAccessor74.invoke(Unknown Source)
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke([email protected]/DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke([email protected]/Method.java:568)
at org.apache.tomcat.jdbc.pool.ProxyConnection.invoke(ProxyConnection.java:131)
at org.apache.tomcat.jdbc.pool.JdbcInterceptor.invoke(JdbcInterceptor.java:107)
at org.apache.tomcat.jdbc.pool.interceptor.AbstractCreateStatementInterceptor.invoke(AbstractCreateStatementInterceptor.java:78)
at org.apache.tomcat.jdbc.pool.JdbcInterceptor.invoke(JdbcInterceptor.java:107)
at org.apache.tomcat.jdbc.pool.DisposableConnectionFacade.invoke(DisposableConnectionFacade.java:81)
at jdk.proxy2.$Proxy83.commit(jdk.proxy2/Unknown Source)
at org.hibernate.resource.jdbc.internal.AbstractLogicalConnectionImplementor.commit(AbstractLogicalConnectionImplementor.java:86)
at org.hibernate.resource.transaction.backend.jdbc.internal.JdbcResourceLocalTransactionCoordinatorImpl$TransactionDriverControlImpl.commit(JdbcResourceLocalTransactionCoordinatorImpl.java:282)
at org.hibernate.engine.transaction.internal.TransactionImpl.commit(TransactionImpl.java:101)
at io.dropwizard.hibernate.SessionFactoryHealthCheck.lambda$check$0(SessionFactoryHealthCheck.java:56)
at io.dropwizard.hibernate.SessionFactoryHealthCheck$$Lambda$1617/0x00000008016fe778.call(Unknown Source)
at java.util.concurrent.FutureTask.run([email protected]/FutureTask.java:264)
at java.util.concurrent.ThreadPoolExecutor.runWorker([email protected]/ThreadPoolExecutor.java:1136)
at java.util.concurrent.ThreadPoolExecutor$Worker.run([email protected]/ThreadPoolExecutor.java:635)
at com.codahale.metrics.InstrumentedThreadFactory$InstrumentedRunnable.run(InstrumentedThreadFactory.java:66)
at java.lang.Thread.run([email protected]/Thread.java:833)
Locked ownable synchronizers:
- <0x00000000cdda95c8> (a java.util.concurrent.ThreadPoolExecutor$Worker)
We have tried use of TLS1.2 , read timeout , all the abandoned settings without any success