Jersey threads stuck in `WAITING`
Thread Name:catalina-exec-367 ID:982 Time:Mon Nov 18 14:04:09 PST 2019 State:WAITING Priority:5 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) org.apache.http.pool.AbstractConnPool.getPoolEntryBlocking(AbstractConnPool.java:379) org.apache.http.pool.AbstractConnPool.access$200(AbstractConnPool.java:69) org.apache.http.pool.AbstractConnPool$2.get(AbstractConnPool.java:245) org.apache.http.pool.AbstractConnPool$2.get(AbstractConnPool.java:193) org.apache.http.impl.conn.PoolingHttpClientConnectionManager.leaseConnection(PoolingHttpClientConnectionManager.java:276) org.apache.http.impl.conn.PoolingHttpClientConnectionManager$1.get(PoolingHttpClientConnectionManager.java:263) org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:190) org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184) org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88) org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110) org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184) org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:71) org.glassfish.jersey.apache.connector.ApacheConnector.apply(ApacheConnector.java:478) org.glassfish.jersey.client.ClientRuntime.invoke(ClientRuntime.java:254) org.glassfish.jersey.client.JerseyInvocation.lambda$invoke$2(JerseyInvocation.java:770) org.glassfish.jersey.client.JerseyInvocation$$Lambda$620/1982271609.call(Unknown Source) org.glassfish.jersey.internal.Errors.process(Errors.java:292) org.glassfish.jersey.internal.Errors.process(Errors.java:274) org.glassfish.jersey.internal.Errors.process(Errors.java:205) org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:390) org.glassfish.jersey.client.JerseyInvocation.invoke(JerseyInvocation.java:768) org.glassfish.jersey.client.JerseyInvocation$Builder.method(JerseyInvocation.java:414) org.glassfish.jersey.client.proxy.WebResourceFactory.invoke(WebResourceFactory.java:331) com.sun.proxy.$Proxy140.getDealers(Unknown Source) com.tms.restws.dealerlocator.client.impl.DealerSearchRestClientImpl.getDealers(DealerSearchRestClientImpl.java:62) com.tms.site.dealerlocator.service.impl.DealerLocatorServiceDisImpl.searchDealers(DealerLocatorServiceDisImpl.java:385) com.tms.site.dealerlocator.service.impl.DealerLocatorServiceDisImpl.searchDealers(DealerLocatorServiceDisImpl.java:164) com.tms.site.dealerlocator.controller.DealerLocatorRestWS.searchDealers(DealerLocatorRestWS.java:101) sun.reflect.GeneratedMethodAccessor219.invoke(Unknown Source) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) java.lang.reflect.Method.invoke(Method.java:498) org.jboss.resteasy.core.MethodInjectorImpl.invoke(MethodInjectorImpl.java:139) org.jboss.resteasy.core.ResourceMethodInvoker.invokeOnTarget(ResourceMethodInvoker.java:295) org.jboss.resteasy.core.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:249) org.jboss.resteasy.core.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:236) org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:402) org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:209) org.jboss.resteasy.plugins.server.servlet.ServletContainerDispatcher.service(ServletContainerDispatcher.java:221) org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.service(HttpServletDispatcher.java:56) org.jboss.resteasy.plugins.server.servlet.HttpServletDispatcher.service(HttpServletDispatcher.java:51) javax.servlet.http.HttpServlet.service(HttpServlet.java:741) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) com.tms.restws.filter.JSONPRequestFilter.doFilter(JSONPRequestFilter.java:109) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201) org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) ch.qos.logback.classic.selector.servlet.LoggerContextFilter.doFilter(LoggerContextFilter.java:70) org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193) org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166) org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:199) org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96) org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:493) org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:137) org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:81) org.apache.catalina.valves.RemoteIpValve.invoke(RemoteIpValve.java:679) org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:660) org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87) org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343) org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:798) org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66) org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:808) org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1498) org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) java.lang.Thread.run(Thread.java:748)
ApacheConnector could throw ConnectionClosedException when using httpclient: 4.5.1+ with chunked transfer encodingProperly close the Apache response so that connections can be reusedApacheConnector/PoolingHttpClientConnectionManager doesn't pool at all!
For all the above problems below implementation has resolved my all the problems
Create client with synchronized blocks as below and then make sure from where ever you are calling the proxy request. Close all the response object in finally block of your rest client implementation
package com.tms.restws; import javax.ws.rs.client.Client; import javax.ws.rs.client.ClientBuilder; import javax.ws.rs.client.WebTarget; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.glassfish.jersey.apache.connector.ApacheClientProperties; import org.glassfish.jersey.apache.connector.ApacheConnectorProvider; import org.glassfish.jersey.client.ClientConfig; import org.glassfish.jersey.client.ClientProperties; import org.glassfish.jersey.client.proxy.WebResourceFactory; public class JerseyRestClient { public static int DEFAULT_CONNECTION_TIMEOUT_MS = 30000; public static int DEFAULT_SOCKET_TIMEOUT_MS = 30000; public static int DEFAULT_MAX_TOTAL_CONNECTIONS = 100; public static int DEFAULT_MAX_CONNECTIONS_DEFAULT_PER_ROUTE = 50; private static Client client = null; public static <T> T createProxy(Class<T> clazz, String baseUrl, int connectionTimeoutMs, int socketTimeoutMs, int maxTotalConnections, int maxDefaultConnectionsPerRoute) { DEFAULT_CONNECTION_TIMEOUT_MS = connectionTimeoutMs; DEFAULT_SOCKET_TIMEOUT_MS = socketTimeoutMs; DEFAULT_MAX_TOTAL_CONNECTIONS = maxTotalConnections; DEFAULT_MAX_CONNECTIONS_DEFAULT_PER_ROUTE = maxDefaultConnectionsPerRoute; client = makeClient(); WebTarget target = client.target(baseUrl); return WebResourceFactory.newResource(clazz, target); } public static WebTarget getClient(String baseUrl, int connectionTimeoutMs, int socketTimeoutMs, int maxTotalConnections, int maxDefaultConnectionsPerRoute) { DEFAULT_CONNECTION_TIMEOUT_MS = connectionTimeoutMs; DEFAULT_SOCKET_TIMEOUT_MS = socketTimeoutMs; DEFAULT_MAX_TOTAL_CONNECTIONS = maxTotalConnections; DEFAULT_MAX_CONNECTIONS_DEFAULT_PER_ROUTE = maxDefaultConnectionsPerRoute; client = makeClient(); WebTarget target = client.target(baseUrl); return target; } public static Client makeClient(){ if(client!=null){ return client; } if (client == null) { synchronized (JerseyRestClient.class) { if (client == null) { ClientConfig clientConfig = new ClientConfig(); clientConfig.property(ClientProperties.CONNECT_TIMEOUT, DEFAULT_CONNECTION_TIMEOUT_MS); clientConfig.property(ClientProperties.READ_TIMEOUT, DEFAULT_CONNECTION_TIMEOUT_MS); // For connection pooling using httpclient PoolingHttpClientConnectionManager poolingClientConnectionManager = new PoolingHttpClientConnectionManager(); poolingClientConnectionManager.setMaxTotal(DEFAULT_MAX_TOTAL_CONNECTIONS); poolingClientConnectionManager.setDefaultMaxPerRoute(DEFAULT_MAX_CONNECTIONS_DEFAULT_PER_ROUTE); clientConfig.property(ApacheClientProperties.CONNECTION_MANAGER, poolingClientConnectionManager); clientConfig.connectorProvider(new ApacheConnectorProvider()); // ends connection pooling client = ClientBuilder.newClient(clientConfig); } } } return client; } }
