Serialized document batching

KeywordSearch/solr/server/solr/configsets/AutopsyConfig/conf/schema.xml

@@ -64,6 +64,8 @@
            content_str will be populated with lowercase content by Autopsy.
        2.2 added text_ja type, content_ja and language fields to support Japanese text search
      -->
+	 
+	<schemaFactory class="ClassicIndexSchemaFactory"/>
 
   <types>
     <!-- field type definitions. The "name" attribute is

KeywordSearch/solr/server/solr/configsets/AutopsyConfig/conf/solrconfig.xml

@@ -140,7 +140,7 @@
     -->
 	
     <!-- Maximum time to wait for a write lock (ms) for an IndexWriter. Default: 1000 -->
-     <writeLockTimeout>1000</writeLockTimeout>  
+     <writeLockTimeout>10000</writeLockTimeout>  
 
     <!-- Expert: Enabling compound file will use less files for the index, 
          using fewer file descriptors on the expense of performance decrease. 
@@ -299,7 +299,7 @@
          searcher to be opened to make those changes visible.
       -->
      <autoCommit> 
-       <maxTime>3000000</maxTime>
+       <maxTime>300000</maxTime>
 	   <!-- maxDocs>15000</maxDocs --> 
        <openSearcher>true</openSearcher> 
      </autoCommit>

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED

@@ -238,7 +238,7 @@ Server.request.exception.exception.msg=Could not issue Solr request
 Server.commit.exception.msg=Could not commit index
 Server.addDoc.exception.msg=Could not add document to index via update handler: {0}
 Server.addDoc.exception.msg2=Could not add document to index via update handler: {0}
-Server.addDocBatch.exception.msg=Could not add document batch to index
+Server.addDocBatch.exception.msg=Could not add batched documents to index
 Server.close.exception.msg=Cannot close Core
 Server.close.exception.msg2=Cannot close Core
 Server.solrServerNoPortException.msg=Indexing server could not bind to port {0}, port is not available, consider change the default {1} port.

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java

@@ -73,6 +73,7 @@ import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.util.NamedList;
 import org.openide.modules.InstalledFileLocator;
 import org.openide.modules.Places;
+import org.openide.util.Exceptions;
 import org.openide.util.NbBundle;
 import org.openide.windows.WindowManager;
 import org.sleuthkit.autopsy.casemodule.Case;
@@ -1879,6 +1880,7 @@ public class Server {
         
         private ScheduledThreadPoolExecutor periodicTasksExecutor = null;
         private static final long PERIODIC_BATCH_SEND_INTERVAL_MINUTES = 10;
+        private static final int NUM_BATCH_UPDATE_RETRIES = 10;
 
         private Collection(String name, Case theCase, Index index) throws TimeoutException, InterruptedException, SolrServerException, IOException {
             this.name = name;
@@ -1889,7 +1891,8 @@ public class Server {
             if (caseType == CaseType.SINGLE_USER_CASE) {
                 // get SolrJ client
                 queryClient = getSolrClient("http://localhost:" + localSolrServerPort + "/solr/" + name); // HttpClient
-                indexingClient = getConcurrentClient("http://localhost:" + localSolrServerPort + "/solr/" + name); // ConcurrentClient
+                indexingClient = getSolrClient("http://localhost:" + localSolrServerPort + "/solr/" + name); // HttpClient
+                // ELTODO indexingClient = getConcurrentClient("http://localhost:" + localSolrServerPort + "/solr/" + name); // ConcurrentClient
             } else {
                 // read Solr connection info from user preferences, unless "solrserver.txt" is present
                 queryClient = configureMultiUserConnection(theCase, index, name);
@@ -1996,13 +1999,13 @@ public class Server {
                 throw new SolrServerException(NbBundle.getMessage(this.getClass(), "Server.commit.exception.msg"), ex);
             }
 
-            try {
+            /* ELTODO try {
                 //commit and block
                 indexingClient.commit(true, true);
             } catch (IOException e) {
                 logger.log(Level.WARNING, "Could not commit index. ", e); //NON-NLS
                 throw new SolrServerException(NbBundle.getMessage(this.getClass(), "Server.commit.exception.msg"), e);
-            }
+            }*/
         }
 
         private void deleteDataSource(Long dsObjId) throws IOException, SolrServerException {
@@ -2047,13 +2050,50 @@ public class Server {
          *
          * @throws KeywordSearchModuleException
          */
-        private void sendBufferedDocs(List<SolrInputDocument> docBuffer) throws KeywordSearchModuleException {
+        // ELTODO SYNCHRONIZATION
+        private synchronized void sendBufferedDocs(List<SolrInputDocument> docBuffer) throws KeywordSearchModuleException {
             
             if (docBuffer.isEmpty()) {
                 return;
             }
-            
+
             try {
+                boolean doRetry = false;
+                for (int reTryAttempt = 0; reTryAttempt < NUM_BATCH_UPDATE_RETRIES; reTryAttempt++) {
+                    try {
+                        doRetry = false;
+                        indexingClient.add(docBuffer);
+                    } catch (Exception ex) {
+                        if (reTryAttempt >= NUM_BATCH_UPDATE_RETRIES) {
+                            logger.log(Level.SEVERE, "Could not add buffered documents to index", ex); //NON-NLS
+                            throw new KeywordSearchModuleException(
+                                    NbBundle.getMessage(this.getClass(), "Server.addDocBatch.exception.msg"), ex); //NON-NLS
+                        } else {
+                            logger.log(Level.SEVERE, "Unable to send document batch to Solr. Re-trying...", ex); //NON-NLS
+                            try {
+                                Thread.sleep(10000L);
+                            } catch (InterruptedException ex1) {
+                                throw new KeywordSearchModuleException(
+                                    NbBundle.getMessage(this.getClass(), "Server.addDocBatch.exception.msg"), ex1); //NON-NLS
+                            }
+                            doRetry = true;
+                        }
+                    }
+                    if (!doRetry) {
+                        if (reTryAttempt > 0) {
+                            logger.log(Level.INFO, "Batch update suceeded after {0} re-try", reTryAttempt); //NON-NLS
+                        }
+                        return;
+                    }
+                }
+                // if we are here, it means all re-try attempts failed
+                logger.log(Level.SEVERE, "Unable to send document batch to Solr. All re-try attempts failed!"); //NON-NLS
+            } finally {
+                docBuffer.clear();
+            }
+        
+
+            /* ELTODO try {
                 indexingClient.add(docBuffer);
             } catch (SolrServerException | RemoteSolrException | IOException ex) {
                 logger.log(Level.SEVERE, "Could not add buffered documents to index", ex); //NON-NLS
@@ -2061,7 +2101,7 @@ public class Server {
                         NbBundle.getMessage(this.getClass(), "Server.addDocBatch.exception.msg"), ex); //NON-NLS
             } finally {
                 docBuffer.clear();
-            }
+            }*/
         }
 
         /**