solr 4.1 snappuller replication failure in master slave setup - solr

Question

We have a master slave setup for solr with solr version 4.1.
We are experiencing errors during replication on the slave. It appears that this happens when there is replication under load, but we cannot consistently reproduce this error. When replication failure occurs, it leave the slave in a bad state (the slave continuously throws exception when trying to replicate on the next attempt).
In addition, temporary index directory under the data folder are sometimes left behind and not cleaned up.
Currently solrconfig configured to use SimpleFSDirectoryFactory.
Any help would be appreciated. Stack trace available below:
09 Aug 2013 17:29:02,292 ERROR snapPuller-27-thread-1 org.apache.solr.handler.ReplicationHandler - SnapPull failed :org.apache.solr.common.SolrException: Error opening new searcher
at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1423)
at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1535)
at org.apache.solr.handler.SnapPuller.doCommit(SnapPuller.java:649)
at org.apache.solr.handler.SnapPuller.fetchLatestIndex(SnapPuller.java:463)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:274)
at org.apache.solr.handler.SnapPuller$1.run(SnapPuller.java:224)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:351)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:722)
Caused by: java.io.EOFException: read past EOF: SimpleFSIndexInput(path="/opt/app/solr/home-search/multicore/product/data/index.20130809172900006/_p_2.del")
at org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:266)
at org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:51)
at org.apache.lucene.store.DataInput.readInt(DataInput.java:84)
at org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:181)
at org.apache.lucene.codecs.lucene40.BitVector.<init>(BitVector.java:346)
at org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat.readLiveDocs(Lucene40LiveDocsFormat.java:90)
at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:62)
at org.apache.lucene.index.ReadersAndLiveDocs.getReader(ReadersAndLiveDocs.java:121)
at org.apache.lucene.index.ReadersAndLiveDocs.getReadOnlyClone(ReadersAndLiveDocs.java:218)
at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:96)
at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:369)
at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:257)
at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:249)
at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1361)
... 13 more
...
09 Aug 2013 17:29:02,408 ERROR snapPuller-263-thread-1 org.apache.solr.handler.ReplicationHandler - SnapPull failed :org.apache.solr.common.SolrException: Index fetch failed :
at org.apache.solr.handler.SnapPuller.fetchLatestIndex(SnapPuller.java:476)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:274)
at org.apache.solr.handler.SnapPuller$1.run(SnapPuller.java:224)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:351)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:722)
Caused by: java.lang.IllegalArgumentException: Unknown directory: org.apache.lucene.store.SimpleFSDirectory#/opt/app/solr/home-search/multicore/product/data/index.20130809172900006
at org.apache.solr.core.CachingDirectoryFactory.addCloseListener(CachingDirectoryFactory.java:87)
at org.apache.solr.handler.SnapPuller.fetchLatestIndex(SnapPuller.java:434)
... 10 more
...
09 Aug 2013 17:27:00,006 WARN snapPuller-191-thread-1 org.apache.solr.handler.SnapPuller - Exception while updating statistics
java.lang.RuntimeException: Already closed
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:246)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:231)
at org.apache.solr.handler.SnapPuller.logReplicationTimeAndConfFiles(SnapPuller.java:548)
at org.apache.solr.handler.SnapPuller.fetchLatestIndex(SnapPuller.java:488)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:274)
at org.apache.solr.handler.SnapPuller$1.run(SnapPuller.java:224)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:351)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:722)
09 Aug 2013 17:27:00,007 ERROR snapPuller-191-thread-1 org.apache.solr.handler.ReplicationHandler - SnapPull failed :java.lang.RuntimeException: Already closed
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:246)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:231)
at org.apache.solr.handler.SnapPuller.fetchLatestIndex(SnapPuller.java:380)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:274)
at org.apache.solr.handler.SnapPuller$1.run(SnapPuller.java:224)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:351)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:722)