We have a Spring application that uses Apache DBCP for connection pooling. Our app has been in production for almost a year with no problems. However, during the past few weeks, the application has frozen a number of times. (By frozen, I mean no requests are handled.) The problem is temporarily fixed by restarting the Glassfish domain that hosts the app.
We are monitoring the memory (heap + nonheap) and CPU utilization of this application. Nothing seems out of the ordinary here. We have noticed however, that when we restart the domain, we get exception messages similar to the one at the bottom of this post in our log files.
I point out that we are using DBCP, because each of this NPEs reports the borrowObject method at the top of the stack. Each of them occurs within the same millisecond during any application shutdown.
We have noticed that there are a couple of these happening in the past (a shutdown with such an NPE), but when the application freezes, there are always five. This, we don't believe to be coincidental, is the same number of HTTPSSLWorkerThreads handling requests to the application.
If anyone knows what is causing this, and how to fix it, help would be greatly appreciated.
[#|2009-09-27T12:17:12.209-0400|SEVERE|sun-appserver9.1|javax.enterprise.system.container.web|_ThreadID=20;_ThreadName=httpSSLWorkerThread-443-4;_RequestID=25b115cb-de08-4ad4-bfdb-f76eb13b4bbe;|StandardWrapperValve[spring-dispatcher]: PWC1406: Servlet.service() for servlet spring-dispatcher threw exception
java.lang.NullPointerException
at org.apache.commons.pool.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:788)
at org.apache.commons.dbcp.PoolingDataSource.getConnection(PoolingDataSource.java:95)
at org.apache.commons.dbcp.BasicDataSource.getConnection(BasicDataSource.java:540)
at org.springframework.jdbc.datasource.DataSourceUtils.doGetConnection(DataSourceUtils.java:113)
at org.springframework.jdbc.datasource.DataSourceUtils.getConnection(DataSourceUtils.java:79)
at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:552)
at org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:616)
at org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:645)
at org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:653)
at org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:685)
at org.springframework.jdbc.core.simple.SimpleJdbcTemplate.query(SimpleJdbcTemplate.java:187)
at stimpl.dao.ProjectDAO.getSearchResults(ProjectDAO.java:153)
at stimpl.service.StimPlService.getSearchResults(StimPlService.java:69)
at stimpl.mvc.search.ShowResultsController.handleRequestInternal(ShowResultsController.java:44)
at org.springframework.web.servlet.mvc.AbstractController.handleRequest(AbstractController.java:153)
at org.springframework.web.servlet.mvc.SimpleControllerHandlerAdapter.handle(SimpleControllerHandlerAdapter.java:48)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:874)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:808)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:476)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:431)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:718)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:831)
at org.apache.catalina.core.ApplicationFilterChain.servletService(ApplicationFilterChain.java:411)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:317)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:198)
at org.acegisecurity.securechannel.ChannelProcessingFilter.doFilter(ChannelProcessingFilter.java:138)
at org.acegisecurity.util.FilterToBeanProxy.doFilter(FilterToBeanProxy.java:98)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:230)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:198)
at org.acegisecurity.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:265)
at org.acegisecurity.intercept.web.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:107)
at org.acegisecurity.intercept.web.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:72)
at org.acegisecurity.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:275)
at org.acegisecurity.ui.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:124)
at org.acegisecurity.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:275)
at org.acegisecurity.providers.anonymous.AnonymousProcessingFilter.doFilter(AnonymousProcessingFilter.java:125)
at org.acegisecurity.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:275)
at org.acegisecurity.wrapper.SecurityContextHolderAwareRequestFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:81)
at org.acegisecurity.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:275)
at org.acegisecurity.ui.AbstractProcessingFilter.doFilter(AbstractProcessingFilter.java:271)
at org.acegisecurity.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:275)
at org.acegisecurity.ui.logout.LogoutFilter.doFilter(LogoutFilter.java:110)
at org.acegisecurity.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:275)
at org.acegisecurity.context.HttpSessionContextIntegrationFilter.doFilter(HttpSessionContextIntegrationFilter.java:249)
at org.acegisecurity.util.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:275)
at org.acegisecurity.util.FilterChainProxy.doFilter(FilterChainProxy.java:149)
at org.acegisecurity.util.FilterToBeanProxy.doFilter(FilterToBeanProxy.java:98)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:230)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:198)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:288)
at org.apache.catalina.core.StandardContextValve.invokeInternal(StandardContextValve.java:271)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:202)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:632)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:577)
at com.sun.enterprise.web.WebPipeline.invoke(WebPipeline.java:94)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:206)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:632)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:577)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:571)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:1080)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:150)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:632)
at org.apache.catalina.core.StandardPipeline.doInvoke(StandardPipeline.java:577)
at org.apache.catalina.core.StandardPipeline.invoke(StandardPipeline.java:571)
at org.apache.catalina.core.ContainerBase.invoke(ContainerBase.java:1080)
at org.apache.coyote.tomcat5.CoyoteAdapter.service(CoyoteAdapter.java:270)
at com.sun.enterprise.web.connector.grizzly.DefaultProcessorTask.invokeAdapter(DefaultProcessorTask.java:637)
at com.sun.enterprise.web.connector.grizzly.DefaultProcessorTask.doProcess(DefaultProcessorTask.java:568)
at com.sun.enterprise.web.connector.grizzly.DefaultProcessorTask.process(DefaultProcessorTask.java:813)
at com.sun.enterprise.web.connector.grizzly.DefaultReadTask.executeProcessorTask(DefaultReadTask.java:339)
at com.sun.enterprise.web.connector.grizzly.ssl.SSLReadTask.process(SSLReadTask.java:440)
at com.sun.enterprise.web.connector.grizzly.ssl.SSLReadTask.doTask(SSLReadTask.java:228)
at com.sun.enterprise.web.connector.grizzly.TaskBase.run(TaskBase.java:265)
at com.sun.enterprise.web.connector.grizzly.ssl.SSLWorkerThread.run(SSLWorkerThread.java:106)
|#]