I really appreciate the help!
Here's what I hope is relevant log info. I had to remove some entries due to the post size limit.
2009-11-02 15:11:14,392 INFO [btpool0-1472] [name=
adminuser@ourdomain.com;mid=5;ip=xxx.xx.xx.x;ua=ZimbraWebClient - FF3.0 (Mac);] soap - ReIndexRequest
2009-11-02 15:11:14,401 INFO [btpool0-1472] [name=
adminuser@ourdomain.com;mid=5;ip=xxx.xx.xx.x;ua=ZimbraWebClient - FF3.0 (Mac);] misc - delegated access: doc=ReIndex, authenticated account=
adminuser@ourdomain.com,
target account=
ouruser@ourdomain.com
2009-11-02 15:11:14,401 INFO [btpool0-1472] [name=
adminuser@ourdomain.com;mid=5;ip=xxx.xx.xx.x;ua=ZimbraWebClient - FF3.0 (Mac);] SoapEngine - handler exception
com.zimbra.common.service.ServiceException: mbox 7c1eb634-93bb-43c0-9a75-c314233f2d68 is not currently running action ReIndex
ExceptionId:btpool0-1472:1257199874401:13d71d6a8a5a01f6
Code:service.NOT_IN_PROGRESS Arg:(mboxId, IID, "7c1eb634-93bb-43c0-9a75-c314233f2d68") Arg:(action, STR, "ReIndex")
at com.zimbra.common.service.ServiceException.NOT_IN_PROGRESS(ServiceException.java:318)
at com.zimbra.cs.service.admin.ReIndex.handle(ReIndex.java:109)
at com.zimbra.soap.SoapEngine.dispatchRequest(SoapEngine.java:417)
at com.zimbra.soap.SoapEngine.dispatch(SoapEngine.java:272)
at com.zimbra.soap.SoapEngine.dispatch(SoapEngine.java:156)
at com.zimbra.soap.SoapServlet.doWork(SoapServlet.java:291)
at com.zimbra.soap.SoapServlet.doPost(SoapServlet.java:212)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:727">HttpServlet.service(HttpServlet.java:727)
at com.zimbra.cs.servlet.ZimbraServlet.service(ZimbraServlet.java:187)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:820">HttpServlet.service(HttpServlet.java:820)
at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:502)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1148)
at org.mortbay.servlet.UserAgentFilter.doFilter(UserAgentFilter.java:81)
at org.mortbay.servlet.GzipFilter.doFilter(GzipFilter.java:129)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1139)
at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:378)
at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:765)
at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
at org.mortbay.jetty.handler.HandlerCollection.handle(HandlerCollection.java:114)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.handler.rewrite.RewriteHandler.handle(RewriteHandler.java:230)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:324)
at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:517)
at org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:868)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:747)
at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:218)
at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:386)
at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:409)
at org.mortbay.thread.BoundedThreadPool$PoolThread.run(BoundedThreadPool.java:451)
2009-11-02 15:11:17,265 INFO [btpool0-1472] [name=
adminuser@ourdomain.com;mid=5;ip=xxx.xx.xx.x;ua=ZimbraWebClient - FF3.0 (Mac);] soap - ReIndexRequest
2009-11-02 15:11:17,288 INFO [btpool0-1472] [name=
adminuser@ourdomain.com;mid=5;ip=xxx.xx.xx.x;ua=ZimbraWebClient - FF3.0 (Mac);] misc - delegated access: doc=ReIndex, authenticated account=
adminuser@ourdomain.com,
target account=
ouruser@ourdomain.com
2009-11-02 15:11:17,722 INFO [ReIndex-1] [name=
ouruser@ourdomain.com;mid=76;] mailbox - IndexDeferredItems(0, 0): Deferred count out of sync - found=38385 in progress=0 (deferred count=100000)
2009-11-02 15:11:18,444 INFO [ReIndex-1] [name=
ouruser@ourdomain.com;mid=76;] HttpMethodDirector - I/O exception (java.net.ConnectException) caught when processing request: Connection refused
2009-11-02 15:11:18,444 INFO [ReIndex-1] [name=
ouruser@ourdomain.com;mid=76;] HttpMethodDirector - Retrying request
2009-11-02 15:11:18,445 WARN [ReIndex-1] [name=
ouruser@ourdomain.com;mid=76;] ParsedMessage - Unable to parse part 9 (linkto_cvs rfi.zip, application/octet-stream) of message with Message-ID
8c0@SOMECOMPUTERNAME>.
com.zimbra.cs.mime.MimeHandlerException: extraction failed
at com.zimbra.cs.mime.handler.ConverterHandler.getContentImpl(ConverterHandler.java:101)
at com.zimbra.cs.mime.MimeHandler.getContent(MimeHandler.java:148)
at com.zimbra.cs.mime.ParsedMessage.analyzePart(ParsedMessage.java:1080)
at com.zimbra.cs.mime.ParsedMessage.analyzeNonBodyParts(ParsedMessage.java:423)
at com.zimbra.cs.mime.ParsedMessage.analyzeFully(ParsedMessage.java:457)
at com.zimbra.cs.mailbox.Message.generateIndexData(Message.java:827)
at com.zimbra.cs.mailbox.IndexHelper.indexItemList(IndexHelper.java:840)
at com.zimbra.cs.mailbox.IndexHelper.indexDeferredItemsInternal(IndexHelper.java:359)
at com.zimbra.cs.mailbox.IndexHelper.indexDeferredItems(IndexHelper.java:277)
at com.zimbra.cs.mailbox.IndexHelper.access$500(IndexHelper.java:54)
at com.zimbra.cs.mailbox.IndexHelper$ReIndexTask.reIndex(IndexHelper.java:487)
at com.zimbra.cs.mailbox.IndexHelper$ReIndexTask.run(IndexHelper.java:417)
at com.zimbra.common.util.ThreadPool$CountedTask.run(ThreadPool.java:136)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
Caused by: com.zimbra.cs.convert.ConversionException: connect failed
... 16 more
Caused by: java.net.ConnectException: Connection refused
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.PlainSocketImpl.doConnect(PlainSocketImpl.java:333)
at java.net.PlainSocketImpl.connectToAddress(PlainSocketImpl.java:195)
at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:182)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:366)
at java.net.Socket.connect(Socket.java:525)
at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.commons.httpclient.protocol.ReflectionSocketFactory.createSocket(ReflectionSocketFactory.java:139)
at org.apache.commons.httpclient.protocol.DefaultProtocolSocketFactory.createSocket(DefaultProtocolSocketFactory.java:124)
at org.apache.commons.httpclient.HttpConnection.open(HttpConnection.java:706)
at org.apache.commons.httpclient.MultiThreadedHttpConnectionManager$HttpConnectionAdapter.open(MultiThreadedHttpConnectionManager.java:1321)
at org.apache.commons.httpclient.HttpMethodDirector.executeWithRetry(HttpMethodDirector.java:386)
at org.apache.commons.httpclient.HttpMethodDirector.executeMethod(HttpMethodDirector.java:170)
at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:396)
at org.apache.commons.httpclient.HttpClient.executeMethod(HttpClient.java:324)
at com.zimbra.cs.convert.ConverterClient.call(ConverterClient.java:107)
at com.zimbra.cs.convert.ConverterClient.post(ConverterClient.java:137)
at com.zimbra.cs.convert.ConverterClient.extract(ConverterClient.java:87)
at com.zimbra.cs.mime.handler.ConverterHandler.getContentImpl(ConverterHandler.java:94)
... 15 more
2009-11-02 15:11:18,445 WARN [ReIndex-1] [name=
ouruser@ourdomain.com;mid=76;] ParsedMessage - Attachment will not be indexed.
2009-11-02 15:11:18,451 WARN [ReIndex-1] [name=
ouruser@ourdomain.com;mid=76;] ParsedMessage - Message had analysis errors in 1 parts (Message-Id: , Subject: RE: CVS RFI)
2009-11-02 15:11:18,451 INFO [ReIndex-1] [name=
ouruser@ourdomain.com;mid=76;] indexadd - Temporary error generating index data for item ID: 406. Indexing will be retried
com.zimbra.cs.mailbox.MailItem$TemporaryIndexingException
at com.zimbra.cs.mailbox.Message.generateIndexData(Message.java:830)
at com.zimbra.cs.mailbox.IndexHelper.indexItemList(IndexHelper.java:840)
at com.zimbra.cs.mailbox.IndexHelper.indexDeferredItemsInternal(IndexHelper.java:359)
at com.zimbra.cs.mailbox.IndexHelper.indexDeferredItems(IndexHelper.java:277)
at com.zimbra.cs.mailbox.IndexHelper.access$500(IndexHelper.java:54)
at com.zimbra.cs.mailbox.IndexHelper$ReIndexTask.reIndex(IndexHelper.java:487)
at com.zimbra.cs.mailbox.IndexHelper$ReIndexTask.run(IndexHelper.java:417)
at com.zimbra.common.util.ThreadPool$CountedTask.run(ThreadPool.java:136)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
2009-11-02 15:11:18,451 INFO [ReIndex-1] [name=
ouruser@ourdomain.com;mid=76;] mailbox - Exception from Mailbox.indexItemList
com.zimbra.common.service.ServiceException: system failure: Temporary indexing exception
ExceptionId:ReIndex-1:1257199878451:13d71d6a8a5a01f6
Code:service.FAILURE
at com.zimbra.common.service.ServiceException.FAILURE(ServiceException.java:251)
at com.zimbra.cs.mailbox.IndexHelper.indexItemList(IndexHelper.java:847)
at com.zimbra.cs.mailbox.IndexHelper.indexDeferredItemsInternal(IndexHelper.java:359)
at com.zimbra.cs.mailbox.IndexHelper.indexDeferredItems(IndexHelper.java:277)
at com.zimbra.cs.mailbox.IndexHelper.access$500(IndexHelper.java:54)
at com.zimbra.cs.mailbox.IndexHelper$ReIndexTask.reIndex(IndexHelper.java:487)
at com.zimbra.cs.mailbox.IndexHelper$ReIndexTask.run(IndexHelper.java:417)
at com.zimbra.common.util.ThreadPool$CountedTask.run(ThreadPool.java:136)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
Caused by: com.zimbra.cs.mailbox.MailItem$TemporaryIndexingException
at com.zimbra.cs.mailbox.Message.generateIndexData(Message.java:830)
at com.zimbra.cs.mailbox.IndexHelper.indexItemList(IndexHelper.java:840)
... 9 more
2009-11-02 15:11:18,451 INFO [ReIndex-1] [name=
ouruser@ourdomain.com;mid=76;] mailbox - Deferred Indexing: submitted 5 items in 1217ms (4.11/sec). (0 items failed to index). IndexDeferredCount now at 38385 NumNotSubmit
ted= 38380
2009-11-02 15:11:18,471 INFO [btpool0-1474] [name=
adminuser@ourdomain.com;mid=5;ip=xxx.xx.xx.x;ua=ZimbraWebClient - FF3.0 (Mac);] soap - ReIndexRequest
2009-11-02 15:11:18,471 INFO [btpool0-1474] [name=
adminuser@ourdomain.com;mid=5;ip=xxx.xx.xx.x;ua=ZimbraWebClient - FF3.0 (Mac);] misc - delegated access: doc=ReIndex, authenticated account=
adminuser@ourdomain.com,
target account=
ouruser@ourdomain.com
2009-11-02 15:11:18,561 INFO [ReIndex-1] [name=
ouruser@ourdomain.com;mid=76;] mailbox - Re-Indexing: Mailbox 76 COMPLETED in 1274ms
2009-11-02 15:11:18,576 INFO [btpool0-1474] [name=
adminuser@ourdomain.com;mid=5;ip=xxx.xx.xx.x;ua=ZimbraWebClient - FF3.0 (Mac);] soap - ReIndexRequest
2009-11-02 15:11:18,576 INFO [btpool0-1474] [name=
adminuser@ourdomain.com;mid=5;ip=xxx.xx.xx.x;ua=ZimbraWebClient - FF3.0 (Mac);] misc - delegated access: doc=ReIndex, authenticated account=
adminuser@ourdomain.com,
target account=
ouruser@ourdomain.com
2009-11-02 15:11:18,576 INFO [btpool0-1474] [name=
adminuser@ourdomain.com;mid=5;ip=xxx.xx.xx.x;ua=ZimbraWebClient - FF3.0 (Mac);] SoapEngine - handler exception
com.zimbra.common.service.ServiceException: mbox 7c1eb634-93bb-43c0-9a75-c314233f2d68 is not currently running action ReIndex
ExceptionId:btpool0-1474:1257199878576:13d71d6a8a5a01f6
Code:service.NOT_IN_PROGRESS Arg:(mboxId, IID, "7c1eb634-93bb-43c0-9a75-c314233f2d68") Arg:(action, STR, "ReIndex")
at com.zimbra.common.service.ServiceException.NOT_IN_PROGRESS(ServiceException.java:318)
at com.zimbra.cs.service.admin.ReIndex.handle(ReIndex.java:109)
at com.zimbra.soap.SoapEngine.dispatchRequest(SoapEngine.java:417)
at com.zimbra.soap.SoapEngine.dispatch(SoapEngine.java:272)
at com.zimbra.soap.SoapEngine.dispatch(SoapEngine.java:156)
at com.zimbra.soap.SoapServlet.doWork(SoapServlet.java:291)
at com.zimbra.soap.SoapServlet.doPost(SoapServlet.java:212)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:727">HttpServlet.service(HttpServlet.java:727)
at com.zimbra.cs.servlet.ZimbraServlet.service(ZimbraServlet.java:187)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:820">HttpServlet.service(HttpServlet.java:820)
at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:502)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1148)
at org.mortbay.servlet.UserAgentFilter.doFilter(UserAgentFilter.java:81)
at org.mortbay.servlet.GzipFilter.doFilter(GzipFilter.java:129)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1139)
at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:378)
at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:181)
at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:765)
at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:417)
at org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
at org.mortbay.jetty.handler.HandlerCollection.handle(HandlerCollection.java:114)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.handler.rewrite.RewriteHandler.handle(RewriteHandler.java:230)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:324)
at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:517)
at org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:868)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:747)
at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:218)
at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:386)
at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:409)
at org.mortbay.thread.BoundedThreadPool$PoolThread.run(BoundedThreadPool.java:451)