Thanks for your response. That is correct, they are both stand alone servers. I am using Zextras as well.
This is the process that is causing the high load:
/opt/zimbra/common/bin/java -Dfile.encoding=UTF-8 -server -Dhttps.protocols=TLSv1,TLSv1.1,TLSv1.2 -Djdk.tls.client.protocols=TLSv1,TLSv1.1,TLSv1.2 -Djava.awt.headless=true -Dsun.net.inetaddr.ttl=60 -Dorg.apache.jasper.compiler.disablejsr199=true -XX:+UseConcMarkSweepGC -XX:SoftRefLRUPolicyMSPerMB=1 -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:-OmitStackTraceInFastThrow -Xloggc:/opt/zimbra/log/gc.log -XX:-UseGCLogFileRotation -XX:NumberOfGCLogFiles=20 -XX:GCLogFileSize=4096K -Djava.net.preferIPv4Stack=true -Xss256k -Xms6389m -Xmx6389m -Xmn3194m -Djava.io.tmpdir=/opt/zimbra/mailboxd/work -Djava.library.path=/opt/zimbra/lib -Djava.endorsed.dirs=/opt/zimbra/mailboxd/common/endorsed -Dzimbra.config=/opt/zimbra/conf/localconfig.xml -Djetty.base=/opt/zimbra/mailboxd -Djetty.home=/opt/zimbra/common/jetty_home -DSTART=/opt/zimbra/mailboxd/etc/start.config -jar /opt/zimbra/common/jetty_home/start.jar --module=zimbra,server,servlet,servlets,jsp,jstl,jmx,resources,websocket,ext,plus,rewrite,monitor,continuation,webapp,setuid jetty.home=/opt/zimbra/common/jetty_home jetty.base=/opt/zimbra/mailboxd /opt/zimbra/mailboxd/etc/jetty.xml
The server is a 32 GB and processor is Intel(R) Xeon(R) CPU E3-1225 V2 @ 3.20GHz.
No raid issues reporting and the system has 32 GB of swap available.
zmproxyctl restart - no change in load
zmmailboxdctl restart - killed the process above and restarted services (from email notifications) zimbraAdmin, mailboxd, mailbox, service, zimlet, zimbra
I think I may have tracked it down to a single mailbox through reading the zmmailboxd.out. It's using IMAP, and threw the below error which seemed to start other errors related to locking.
2018-01-03 01:19:27,793 INFO [ImapSSLServer-1] [name=
sales@xxx.ca;mid=6;ip=192.99.xxx.xxx;oip=192.99.xxx.xxx3;via=192.99.xxx.xxx(nginx/1.7.1);ua=ZimbraImapDataSource/8.8.5_GA_1894;] imap - selected folder INBOX/Stxxxx at com.zimbra.soap.SoapServlet.doPost(SoapServlet.java:213)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
at com.zimbra.cs.servlet.ZimbraServlet.service(ZimbraServlet.java:206)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:821)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1685)
at com.zimbra.cs.servlet.CsrfFilter.doFilter(CsrfFilter.java:169)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1668)
at com.zimbra.cs.servlet.RequestStringFilter.doFilter(RequestStringFilter.java:54)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1668)
at com.zimbra.cs.servlet.SetHeaderFilter.doFilter(SetHeaderFilter.java:59)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1668)
at com.zimbra.cs.servlet.ETagHeaderFilter.doFilter(ETagHeaderFilter.java:47)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1668)
at com.zimbra.cs.servlet.ContextPathBasedThreadPoolBalancerFilter.doFilter(ContextPathBasedThreadPoolBalancerFilter.java:107)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1668)
at com.zimbra.cs.servlet.ZimbraQoSFilter.doFilter(ZimbraQoSFilter.java:116)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1668)
at com.zimbra.cs.servlet.ZimbraInvalidLoginFilter.doFilter(ZimbraInvalidLoginFilter.java:117)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1668)
at org.eclipse.jetty.servlets.DoSFilter.doFilterChain(DoSFilter.java:473)
at org.eclipse.jetty.servlets.DoSFilter.doFilter(DoSFilter.java:318)
at org.eclipse.jetty.servlets.DoSFilter.doFilter(DoSFilter.java:288)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1668)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:524)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1158)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1090)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:109)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:119)
at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:318)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:437)
at org.eclipse.jetty.server.handler.DebugHandler.handle(DebugHandler.java:84)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:119)
at org.eclipse.jetty.server.Server.handle(Server.java:517)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:306)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:242)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:261)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:192)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:261)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:75)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:213)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
at java.lang.Thread.run(Thread.java:748)
Lock Waiter - qtp998351292-4769:https:https://mail.xxx.com/service/soap/SearchRequest prio=5 id=4769 state=TIMED_WAITING
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.tryLock(ReentrantReadWriteLock.java:871)
at com.zimbra.cs.mailbox.MailboxLock.tryLockWithTimeout(MailboxLock.java:116)
at com.zimbra.cs.mailbox.MailboxLock.lock(MailboxLock.java:194)
at com.zimbra.cs.mailbox.Mailbox.beginTransaction(Mailbox.java:1759)
at com.zimbra.cs.mailbox.Mailbox.beginReadTransaction(Mailbox.java:1735)
at com.zimbra.cs.mailbox.Mailbox.getItemById(Mailbox.java:2864)
at com.zimbra.cs.mailbox.Mailbox.getItemById(Mailbox.java:2856)
at com.zimbra.cs.mailbox.Mailbox.getFolderById(Mailbox.java:4092)
at com.zimbra.cs.index.query.InQuery.create(InQuery.java:68)
at com.zimbra.cs.index.query.parser.QueryParser.createQuery(QueryParser.java:577)
at com.zimbra.cs.index.query.parser.QueryParser.toTerm(QueryParser.java:427)
at com.zimbra.cs.index.query.parser.QueryParser.toTextClause(QueryParser.java:399)
at com.zimbra.cs.index.query.parser.QueryParser.toClause(QueryParser.java:363)
at com.zimbra.cs.index.query.parser.QueryParser.toQuery(QueryParser.java:342)
at com.zimbra.cs.index.query.parser.QueryParser.parse(QueryParser.java:305)
at com.zimbra.cs.index.ZimbraQuery.<init>(ZimbraQuery.java:378)
at com.zimbra.cs.mailbox.MailboxIndex.search(MailboxIndex.java:166)
at com.zimbra.cs.service.mail.Search.handle(Search.java:111)
at com.zimbra.soap.SoapEngine.dispatchRequest(SoapEngine.java:607)
at com.zimbra.soap.SoapEngine.dispatch(SoapEngine.java:460)
at com.zimbra.soap.SoapEngine.dispatch(SoapEngine.java:273)