mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Serialized document batching

Browse Source
This commit is contained in:
Eugene Livis 2020-08-20 09:46:57 -04:00
parent de08b5e909
commit bc64b01879
4 changed files with 51 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
KeywordSearch/solr/server/solr/configsets/AutopsyConfig/conf/schema.xml
View File

@ -64,6 +64,8 @@
content_str will be populated with lowercase content by Autopsy.
2.2 added text_ja type, content_ja and language fields to support Japanese text search
-->
<schemaFactory class="ClassicIndexSchemaFactory"/>
<types>
<!-- field type definitions. The "name" attribute is

4
KeywordSearch/solr/server/solr/configsets/AutopsyConfig/conf/solrconfig.xml
View File

@ -140,7 +140,7 @@
-->
<!-- Maximum time to wait for a write lock (ms) for an IndexWriter. Default: 1000 -->
<writeLockTimeout>1000</writeLockTimeout>
<writeLockTimeout>10000</writeLockTimeout>
<!-- Expert: Enabling compound file will use less files for the index,
using fewer file descriptors on the expense of performance decrease.
@ -299,7 +299,7 @@
searcher to be opened to make those changes visible.
-->
<autoCommit>
<maxTime>3000000</maxTime>
<maxTime>300000</maxTime>
<!-- maxDocs>15000</maxDocs -->
<openSearcher>true</openSearcher>
</autoCommit>

2
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED
View File

@ -238,7 +238,7 @@ Server.request.exception.exception.msg=Could not issue Solr request
Server.commit.exception.msg=Could not commit index
Server.addDoc.exception.msg=Could not add document to index via update handler: {0}
Server.addDoc.exception.msg2=Could not add document to index via update handler: {0}
Server.addDocBatch.exception.msg=Could not add document batch to index
Server.addDocBatch.exception.msg=Could not add batched documents to index
Server.close.exception.msg=Cannot close Core
Server.close.exception.msg2=Cannot close Core
Server.solrServerNoPortException.msg=Indexing server could not bind to port {0}, port is not available, consider change the default {1} port.

52
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java
View File

@ -73,6 +73,7 @@ import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.util.NamedList;
import org.openide.modules.InstalledFileLocator;
import org.openide.modules.Places;
import org.openide.util.Exceptions;
import org.openide.util.NbBundle;
import org.openide.windows.WindowManager;
import org.sleuthkit.autopsy.casemodule.Case;
@ -1879,6 +1880,7 @@ public class Server {
private ScheduledThreadPoolExecutor periodicTasksExecutor = null;
private static final long PERIODIC_BATCH_SEND_INTERVAL_MINUTES = 10;
private static final int NUM_BATCH_UPDATE_RETRIES = 10;
private Collection(String name, Case theCase, Index index) throws TimeoutException, InterruptedException, SolrServerException, IOException {
this.name = name;
@ -1889,7 +1891,8 @@ public class Server {
if (caseType == CaseType.SINGLE_USER_CASE) {
// get SolrJ client
queryClient = getSolrClient("http://localhost:" + localSolrServerPort + "/solr/" + name); // HttpClient
indexingClient = getConcurrentClient("http://localhost:" + localSolrServerPort + "/solr/" + name); // ConcurrentClient
indexingClient = getSolrClient("http://localhost:" + localSolrServerPort + "/solr/" + name); // HttpClient
// ELTODO indexingClient = getConcurrentClient("http://localhost:" + localSolrServerPort + "/solr/" + name); // ConcurrentClient
} else {
// read Solr connection info from user preferences, unless "solrserver.txt" is present
queryClient = configureMultiUserConnection(theCase, index, name);
@ -1996,13 +1999,13 @@ public class Server {
throw new SolrServerException(NbBundle.getMessage(this.getClass(), "Server.commit.exception.msg"), ex);
}
try {
/* ELTODO try {
//commit and block
indexingClient.commit(true, true);
} catch (IOException e) {
logger.log(Level.WARNING, "Could not commit index. ", e); //NON-NLS
throw new SolrServerException(NbBundle.getMessage(this.getClass(), "Server.commit.exception.msg"), e);
}
}*/
}
private void deleteDataSource(Long dsObjId) throws IOException, SolrServerException {
@ -2047,13 +2050,50 @@ public class Server {
*
* @throws KeywordSearchModuleException
*/
private void sendBufferedDocs(List<SolrInputDocument> docBuffer) throws KeywordSearchModuleException {
// ELTODO SYNCHRONIZATION
private synchronized void sendBufferedDocs(List<SolrInputDocument> docBuffer) throws KeywordSearchModuleException {
if (docBuffer.isEmpty()) {
return;
}
try {
boolean doRetry = false;
for (int reTryAttempt = 0; reTryAttempt < NUM_BATCH_UPDATE_RETRIES; reTryAttempt++) {
try {
doRetry = false;
indexingClient.add(docBuffer);
} catch (Exception ex) {
if (reTryAttempt >= NUM_BATCH_UPDATE_RETRIES) {
logger.log(Level.SEVERE, "Could not add buffered documents to index", ex); //NON-NLS
throw new KeywordSearchModuleException(
NbBundle.getMessage(this.getClass(), "Server.addDocBatch.exception.msg"), ex); //NON-NLS
} else {
logger.log(Level.SEVERE, "Unable to send document batch to Solr. Re-trying...", ex); //NON-NLS
try {
Thread.sleep(10000L);
} catch (InterruptedException ex1) {
throw new KeywordSearchModuleException(
NbBundle.getMessage(this.getClass(), "Server.addDocBatch.exception.msg"), ex1); //NON-NLS
}
doRetry = true;
}
}
if (!doRetry) {
if (reTryAttempt > 0) {
logger.log(Level.INFO, "Batch update suceeded after {0} re-try", reTryAttempt); //NON-NLS
}
return;
}
}
// if we are here, it means all re-try attempts failed
logger.log(Level.SEVERE, "Unable to send document batch to Solr. All re-try attempts failed!"); //NON-NLS
} finally {
docBuffer.clear();
}
/* ELTODO try {
indexingClient.add(docBuffer);
} catch (SolrServerException | RemoteSolrException | IOException ex) {
logger.log(Level.SEVERE, "Could not add buffered documents to index", ex); //NON-NLS
@ -2061,7 +2101,7 @@ public class Server {
NbBundle.getMessage(this.getClass(), "Server.addDocBatch.exception.msg"), ex); //NON-NLS
} finally {
docBuffer.clear();
}
}*/
}
/**