mstoeck3/autopsy-flatpak
1
0
Fork 0
mirror of https://github.com/overcuriousity/autopsy-flatpak.git synced 2025-07-06 21:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1427 from esaunders/solr_spin

Browse Source 
Moved Solr query outside for loop in LuceneQuery.performLuceneQuery().
This commit is contained in:
esaunders 2015-07-08 13:55:29 -04:00
commit 3581be77a8
5 changed files with 77 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties
View File

@ -66,6 +66,7 @@ ExtractedContentViewer.getSolrContent.knownFileMsg=<p style\=''font-style\:itali
ExtractedContentViewer.getSolrContent.noTxtYetMsg=<p style\=''font-style\:italic''>{0} does not have text in the index.<br/>It may have no text, not been analyzed yet, or keyword search was not enabled during ingest.</p> ExtractedContentViewer.getSolrContent.noTxtYetMsg=<p style\=''font-style\:italic''>{0} does not have text in the index.<br/>It may have no text, not been analyzed yet, or keyword search was not enabled during ingest.</p>
ExtractedContentViewer.getSolrContent.txtBodyItal=<span style\=''font-style\:italic''>{0}</span> ExtractedContentViewer.getSolrContent.txtBodyItal=<span style\=''font-style\:italic''>{0}</span>
HighlightedMatchesSource.getMarkup.noMatchMsg=<html><pre><span style\\\\\='background\\\\\:yellow'>There were no keyword hits on this page. <br />Keyword could have been in file name. <br />Advance to another page for hits or choose Extracted Text to view original text..</span></pre></html> HighlightedMatchesSource.getMarkup.noMatchMsg=<html><pre><span style\\\\\='background\\\\\:yellow'>There were no keyword hits on this page. <br />Keyword could have been in file name. <br />Advance to another page for hits or choose Extracted Text to view original text..</span></pre></html>
HighlightedMatchesSource.getMarkup.queryFailedMsg=<html><pre><span style\\\\\='background\\\\\:yellow'>Failed to retrieve keyword hit results. <br />Confirm that Autopsy can connect to the Solr server. <br /></span></pre></html>
HighlightedMatchesSource.toString=Search Results HighlightedMatchesSource.toString=Search Results
Installer.reportPortError=Indexing server port {0} is not available. Check if your security software does not block {1} and consider changing {2} in {3} property file in the application user folder. Then try rebooting your system if another process was causing the conflict. Installer.reportPortError=Indexing server port {0} is not available. Check if your security software does not block {1} and consider changing {2} in {3} property file in the application user folder. Then try rebooting your system if another process was causing the conflict.
Installer.reportStopPortError=Indexing server stop port {0} is not available. Consider changing {1} in {2} property file in the application user folder. Installer.reportStopPortError=Indexing server stop port {0} is not available. Consider changing {1} in {2} property file in the application user folder.
@ -212,11 +213,12 @@ Server.openCore.exception.alreadyOpen.msg=Already an open Core\! Explicitely clo
Server.queryNumIdxFiles.exception.msg=Error querying number of indexed files, Server.queryNumIdxFiles.exception.msg=Error querying number of indexed files,
Server.queryNumIdxChunks.exception.msg=Error querying number of indexed chunks, Server.queryNumIdxChunks.exception.msg=Error querying number of indexed chunks,
Server.queryNumIdxDocs.exception.msg=Error querying number of indexed documents, Server.queryNumIdxDocs.exception.msg=Error querying number of indexed documents,
Server.queryIsIdxd.exception.msg=Error checkign if content is indexed, Server.queryIsIdxd.exception.msg=Error checking if content is indexed,
Server.queryNumFileChunks.exception.msg=Error getting number of file chunks, Server.queryNumFileChunks.exception.msg=Error getting number of file chunks,
Server.query.exception.msg=Error running query\: {0} Server.query.exception.msg=Error running query\: {0}
Server.query2.exception.msg=Error running query\: {0} Server.query2.exception.msg=Error running query\: {0}
Server.queryTerms.exception.msg=Error running terms query\: {0} Server.queryTerms.exception.msg=Error running terms query\: {0}
Server.connect.exception.msg=Failed to connect to Solr server\:
Server.openCore.exception.msg=Core open requested, but server not yet running Server.openCore.exception.msg=Core open requested, but server not yet running
Server.openCore.exception.cantOpen.msg=Could not open Core Server.openCore.exception.cantOpen.msg=Could not open Core
Server.openCore.exception.cantOpen.msg2=Could not open Core Server.openCore.exception.cantOpen.msg2=Could not open Core

9
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedTextMarkup.java
View File

@ -34,7 +34,6 @@ import org.apache.solr.client.solrj.response.QueryResponse;
import org.sleuthkit.autopsy.coreutils.Version; import org.sleuthkit.autopsy.coreutils.Version;
import org.sleuthkit.autopsy.datamodel.TextMarkupLookup; import org.sleuthkit.autopsy.datamodel.TextMarkupLookup;
import org.sleuthkit.autopsy.keywordsearch.KeywordQueryFilter.FilterType; import org.sleuthkit.autopsy.keywordsearch.KeywordQueryFilter.FilterType;
import org.sleuthkit.datamodel.Content;
/** /**
* Highlights hits for a given document. Knows about pages and such for the content viewer. * Highlights hits for a given document. Knows about pages and such for the content viewer.
@ -379,12 +378,8 @@ class HighlightedTextMarkup implements TextMarkup, TextMarkupLookup {
return "<html><pre>" + highlightedContent + "</pre></html>"; //NON-NLS return "<html><pre>" + highlightedContent + "</pre></html>"; //NON-NLS
} }
} catch (NoOpenCoreException ex) { } catch (NoOpenCoreException | KeywordSearchModuleException ex) {
logger.log(Level.WARNING, "Couldn't query markup for page: " + currentPage, ex); //NON-NLS return NbBundle.getMessage(this.getClass(), "HighlightedMatchesSource.getMarkup.queryFailedMsg");
return "";
} catch (KeywordSearchModuleException ex) {
logger.log(Level.WARNING, "Could not query markup for page: " + currentPage, ex); //NON-NLS
return "";
} }
} }

74
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java
View File

@ -33,14 +33,15 @@ import org.apache.solr.client.solrj.SolrRequest.METHOD;
import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrDocumentList;
import org.openide.util.NbBundle;
import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coreutils.EscapeUtil; import org.sleuthkit.autopsy.coreutils.EscapeUtil;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
import org.sleuthkit.autopsy.coreutils.Version; import org.sleuthkit.autopsy.coreutils.Version;
import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskException; import org.sleuthkit.datamodel.TskException;
@ -202,48 +203,51 @@ class LuceneQuery implements KeywordSearchQuery {
final Server solrServer = KeywordSearch.getServer(); final Server solrServer = KeywordSearch.getServer();
SolrQuery q = createAndConfigureSolrQuery(snippets); SolrQuery q = createAndConfigureSolrQuery(snippets);
QueryResponse response;
SolrDocumentList resultList;
Map<String, Map<String, List<String>>> highlightResponse;
Set<SolrDocument> uniqueSolrDocumentsWithHits;
try {
response = solrServer.query(q, METHOD.POST);
resultList = response.getResults();
// objectId_chunk -> "text" -> List of previews
highlightResponse = response.getHighlighting();
// get the unique set of files with hits
uniqueSolrDocumentsWithHits = filterOneHitPerDocument(resultList);
}
catch (KeywordSearchModuleException ex) {
logger.log(Level.SEVERE, "Error executing Lucene Solr Query: " + keywordString, ex); //NON-NLS
MessageNotifyUtil.Notify.error(NbBundle.getMessage(Server.class, "Server.query.exception.msg", keywordString), ex.getCause().getMessage());
return matches;
}
// cycle through results in sets of MAX_RESULTS // cycle through results in sets of MAX_RESULTS
for (int start = 0; !allMatchesFetched; start = start + MAX_RESULTS) { for (int start = 0; !allMatchesFetched; start = start + MAX_RESULTS) {
q.setStart(start); q.setStart(start);
allMatchesFetched = start + MAX_RESULTS >= resultList.getNumFound();
SleuthkitCase sleuthkitCase;
try { try {
QueryResponse response = solrServer.query(q, METHOD.POST); sleuthkitCase = Case.getCurrentCase().getSleuthkitCase();
SolrDocumentList resultList = response.getResults(); } catch (IllegalStateException ex) {
//no case open, must be just closed
// objectId_chunk -> "text" -> List of previews return matches;
Map<String, Map<String, List<String>>> highlightResponse = response.getHighlighting();
// get the unique set of files with hits
Set<SolrDocument> uniqueSolrDocumentsWithHits = filterOneHitPerDocument(resultList);
allMatchesFetched = start + MAX_RESULTS >= resultList.getNumFound();
SleuthkitCase sleuthkitCase;
try {
sleuthkitCase = Case.getCurrentCase().getSleuthkitCase();
} catch (IllegalStateException ex) {
//no case open, must be just closed
return matches;
}
for (SolrDocument resultDoc : uniqueSolrDocumentsWithHits) {
KeywordHit contentHit;
try {
contentHit = createKeywordtHit(resultDoc, highlightResponse, sleuthkitCase);
} catch (TskException ex) {
return matches;
}
matches.add(contentHit);
}
} catch (NoOpenCoreException ex) {
logger.log(Level.WARNING, "Error executing Lucene Solr Query: " + keywordString, ex); //NON-NLS
throw ex;
} catch (KeywordSearchModuleException ex) {
logger.log(Level.WARNING, "Error executing Lucene Solr Query: " + keywordString, ex); //NON-NLS
} }
for (SolrDocument resultDoc : uniqueSolrDocumentsWithHits) {
KeywordHit contentHit;
try {
contentHit = createKeywordtHit(resultDoc, highlightResponse, sleuthkitCase);
} catch (TskException ex) {
return matches;
}
matches.add(contentHit);
}
} }
return matches; return matches;
} }

43
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java
View File

@ -33,7 +33,6 @@ import java.net.ConnectException;
import java.net.ServerSocket; import java.net.ServerSocket;
import java.net.SocketException; import java.net.SocketException;
import java.nio.charset.Charset; import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
@ -65,6 +64,7 @@ import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException;
import org.sleuthkit.autopsy.casemodule.Case.CaseType; import org.sleuthkit.autopsy.casemodule.Case.CaseType;
import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.core.UserPreferences;
import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil;
/** /**
* Handles for keeping track of a Solr server and its cores * Handles for keeping track of a Solr server and its cores
@ -665,6 +665,23 @@ public class Server {
* @return * @return
*/ */
private synchronized Core openCore(Case theCase) throws KeywordSearchModuleException { private synchronized Core openCore(Case theCase) throws KeywordSearchModuleException {
try {
if (theCase.getCaseType() == CaseType.SINGLE_USER_CASE) {
currentSolrServer = this.localSolrServer;
}
else {
String host = UserPreferences.getIndexingServerHost();
String port = UserPreferences.getIndexingServerPort();
currentSolrServer = new HttpSolrServer("http://" + host + ":" + port + "/solr"); //NON-NLS
}
connectToSolrServer(currentSolrServer);
}
catch (SolrServerException | IOException ex) {
MessageNotifyUtil.Notify.error(NbBundle.getMessage(Server.class, "Server.connect.exception.msg"), ex.getCause().getMessage());
throw new KeywordSearchModuleException(NbBundle.getMessage(Server.class, "Server.connect.exception.msg"));
}
String dataDir = getIndexDirPath(theCase); String dataDir = getIndexDirPath(theCase);
String coreName = theCase.getTextIndexName(); String coreName = theCase.getTextIndexName();
return this.openCore(coreName.isEmpty() ? DEFAULT_CORE_NAME : coreName, new File(dataDir), theCase.getCaseType()); return this.openCore(coreName.isEmpty() ? DEFAULT_CORE_NAME : coreName, new File(dataDir), theCase.getCaseType());
@ -696,7 +713,7 @@ public class Server {
* *
* @return int representing number of indexed files * @return int representing number of indexed files
* @throws KeywordSearchModuleException * @throws KeywordSearchModuleException
* @throws NoOpenCoreExceptionn * @throws NoOpenCoreException
*/ */
public int queryNumIndexedFiles() throws KeywordSearchModuleException, NoOpenCoreException { public int queryNumIndexedFiles() throws KeywordSearchModuleException, NoOpenCoreException {
if (currentCore == null) { if (currentCore == null) {
@ -940,6 +957,7 @@ public class Server {
* @return new core * @return new core
*/ */
private Core openCore(String coreName, File dataDir, CaseType caseType) throws KeywordSearchModuleException { private Core openCore(String coreName, File dataDir, CaseType caseType) throws KeywordSearchModuleException {
try { try {
if (!dataDir.exists()) { if (!dataDir.exists()) {
dataDir.mkdirs(); dataDir.mkdirs();
@ -952,14 +970,6 @@ public class Server {
NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg")); NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg"));
} }
if (caseType == CaseType.SINGLE_USER_CASE) {
currentSolrServer = this.localSolrServer;
//createCore.setInstanceDir(instanceDir);
}
else {
currentSolrServer = connectToRemoteSolrServer();
}
if (!isCoreLoaded(coreName)) { if (!isCoreLoaded(coreName)) {
CoreAdminRequest.Create createCore = new CoreAdminRequest.Create(); CoreAdminRequest.Create createCore = new CoreAdminRequest.Create();
createCore.setDataDir(dataDir.getAbsolutePath()); createCore.setDataDir(dataDir.getAbsolutePath());
@ -984,11 +994,14 @@ public class Server {
} }
} }
HttpSolrServer connectToRemoteSolrServer() { /**
String host = UserPreferences.getIndexingServerHost(); * Attempts to connect to the given Solr server.
String port = UserPreferences.getIndexingServerPort(); * @param solrServer
* @throws SolrServerException
return new HttpSolrServer("http://" + host + ":" + port + "/solr"); * @throws IOException
*/
void connectToSolrServer(HttpSolrServer solrServer) throws SolrServerException, IOException {
CoreAdminRequest.getStatus(null, solrServer);
} }
/** /**

9
KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java
View File

@ -22,15 +22,14 @@ import java.io.IOException;
import java.util.HashMap; import java.util.HashMap;
import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute;
import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.datamodel.TskCoreException;
import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService;
import org.apache.solr.common.util.ContentStreamBase.StringStream; import org.apache.solr.common.util.ContentStreamBase.StringStream;
import org.openide.util.Exceptions;
import org.openide.util.lookup.ServiceProvider; import org.openide.util.lookup.ServiceProvider;
import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.core.UserPreferences;
import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.autopsy.datamodel.ContentUtils;
import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.AbstractFile;
import org.sleuthkit.datamodel.Content; import org.sleuthkit.datamodel.Content;
@ -152,8 +151,10 @@ public class SolrSearchService implements KeywordSearchService {
@Override @Override
public boolean canConnectToRemoteSolrServer() { public boolean canConnectToRemoteSolrServer() {
try { try {
HttpSolrServer solrServer = KeywordSearch.getServer().connectToRemoteSolrServer(); String host = UserPreferences.getIndexingServerHost();
CoreAdminRequest.getStatus(null, solrServer); String port = UserPreferences.getIndexingServerPort();
HttpSolrServer solrServer = new HttpSolrServer("http://" + host + ":" + port + "/solr"); //NON-NLS;
KeywordSearch.getServer().connectToSolrServer(solrServer);
} }
catch (SolrServerException | IOException ex) { catch (SolrServerException | IOException ex) {
return false; return false;