From e1b32a9274f9b86db9fe8338f7bd9356463db467 Mon Sep 17 00:00:00 2001 From: Eamonn Saunders Date: Tue, 7 Jul 2015 15:46:58 -0400 Subject: [PATCH 1/3] Updated ChildFactory.createKeys() implementations to not return false when they are unable to create keys. Returning false means "the list has only been partially populated and this method should be called again to batch more keys". --- .../autopsy/datamodel/ExtractedContent.java | 61 +++++++++---------- .../sleuthkit/autopsy/datamodel/FileSize.java | 21 ++----- 2 files changed, 35 insertions(+), 47 deletions(-) diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java b/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java index 728b79fd87..11fac248b4 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/ExtractedContent.java @@ -210,33 +210,31 @@ public class ExtractedContent implements AutopsyVisitableItem { @Override protected boolean createKeys(List list) { - if (skCase == null) { - return false; - } - - try { - List inUse = skCase.getBlackboardArtifactTypesInUse(); - inUse.removeAll(doNotShow); - Collections.sort(inUse, - new Comparator() { - @Override - public int compare(BlackboardArtifact.ARTIFACT_TYPE a, BlackboardArtifact.ARTIFACT_TYPE b) { - return a.getDisplayName().compareTo(b.getDisplayName()); - } - }); - list.addAll(inUse); - // the create node method will get called only for new types - // refresh the counts if we already created them from a previous update - for (BlackboardArtifact.ARTIFACT_TYPE art : inUse) { - TypeNode node = typeNodeList.get(art); - if (node != null) { - node.updateDisplayName(); + if (skCase != null) { + try { + List inUse = skCase.getBlackboardArtifactTypesInUse(); + inUse.removeAll(doNotShow); + Collections.sort(inUse, + new Comparator() { + @Override + public int compare(BlackboardArtifact.ARTIFACT_TYPE a, BlackboardArtifact.ARTIFACT_TYPE b) { + return a.getDisplayName().compareTo(b.getDisplayName()); + } + }); + list.addAll(inUse); + + // the create node method will get called only for new types + // refresh the counts if we already created them from a previous update + for (BlackboardArtifact.ARTIFACT_TYPE art : inUse) { + TypeNode node = typeNodeList.get(art); + if (node != null) { + node.updateDisplayName(); + } } + } catch (TskCoreException ex) { + Logger.getLogger(TypeFactory.class.getName()).log(Level.SEVERE, "Error getting list of artifacts in use: " + ex.getLocalizedMessage()); //NON-NLS } - } catch (TskCoreException ex) { - Logger.getLogger(TypeFactory.class.getName()).log(Level.SEVERE, "Error getting list of artifacts in use: " + ex.getLocalizedMessage()); //NON-NLS - return false; } return true; } @@ -446,15 +444,14 @@ public class ExtractedContent implements AutopsyVisitableItem { @Override protected boolean createKeys(List list) { - if (skCase == null) { - return false; - } - try { - List arts = skCase.getBlackboardArtifacts(type.getTypeID()); - list.addAll(arts); - } catch (TskException ex) { - Logger.getLogger(ArtifactFactory.class.getName()).log(Level.SEVERE, "Couldn't get blackboard artifacts from database", ex); //NON-NLS + if (skCase != null) { + try { + List arts = skCase.getBlackboardArtifacts(type.getTypeID()); + list.addAll(arts); + } catch (TskException ex) { + Logger.getLogger(ArtifactFactory.class.getName()).log(Level.SEVERE, "Couldn't get blackboard artifacts from database", ex); //NON-NLS + } } return true; } diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/FileSize.java b/Core/src/org/sleuthkit/autopsy/datamodel/FileSize.java index 521a529eb7..5febad0a34 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/FileSize.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/FileSize.java @@ -363,11 +363,7 @@ public class FileSize implements AutopsyVisitableItem { @Override protected boolean createKeys(List list) { - List l = runFsQuery(); - if (l == null) { - return false; - } - list.addAll(l); + list.addAll(runFsQuery()); return true; } @@ -386,8 +382,7 @@ public class FileSize implements AutopsyVisitableItem { break; default: - logger.log(Level.SEVERE, "Unsupported filter type to get files by size: {0}", filter); //NON-NLS - return null; + throw new IllegalArgumentException("Unsupported filter type to get files by size: " + filter); //NON-NLS } // ignore unalloc block files query = query + " AND (type != " + TskData.TSK_DB_FILES_TYPE_ENUM.UNALLOC_BLOCKS.getFileType() + ")"; //NON-NLS @@ -398,19 +393,15 @@ public class FileSize implements AutopsyVisitableItem { private List runFsQuery() { List ret = new ArrayList<>(); - String query = makeQuery(filter); - if (query == null) { - return null; - } - try { + String query = makeQuery(filter); + ret = skCase.findAllFilesWhere(query); - } catch (TskCoreException e) { - logger.log(Level.SEVERE, "Error getting files for the file size view using: " + query, e); //NON-NLS + } catch (Exception e) { + logger.log(Level.SEVERE, "Error getting files for the file size view: " + e.getMessage()); //NON-NLS } return ret; - } /** From 8b8685097bc9181024a34c46ddcd879b641a496f Mon Sep 17 00:00:00 2001 From: Ann Priestman Date: Wed, 8 Jul 2015 13:37:20 -0400 Subject: [PATCH 2/3] Add tagging to context menu for artifacts linked to a datasource --- .../autopsy/directorytree/DataResultFilterNode.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java b/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java index f6fed23038..ff0227a9fd 100755 --- a/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java +++ b/Core/src/org/sleuthkit/autopsy/directorytree/DataResultFilterNode.java @@ -237,6 +237,12 @@ public class DataResultFilterNode extends FilterNode { actions.add(AddBlackboardArtifactTagAction.getInstance()); actions.addAll(ContextMenuExtensionPoint.getActions()); } + else{ + // There's no specific file associated with the artifact, but + // we can still tag the artifact itself + actions.add(null); + actions.add(AddBlackboardArtifactTagAction.getInstance()); + } return actions; } From b975f230f9b652ad154974cb84afc9f8eb0283cb Mon Sep 17 00:00:00 2001 From: Eamonn Saunders Date: Wed, 8 Jul 2015 13:52:39 -0400 Subject: [PATCH 3/3] Moved Solr query outside for loop in LuceneQuery.performLuceneQuery(). Modified Server.openCore() to check Solr server connectivity. --- .../autopsy/keywordsearch/Bundle.properties | 4 +- .../keywordsearch/HighlightedTextMarkup.java | 9 +-- .../autopsy/keywordsearch/LuceneQuery.java | 74 ++++++++++--------- .../autopsy/keywordsearch/Server.java | 43 +++++++---- .../keywordsearch/SolrSearchService.java | 9 ++- 5 files changed, 77 insertions(+), 62 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index 0eb71bec79..e96ae82b02 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -66,6 +66,7 @@ ExtractedContentViewer.getSolrContent.knownFileMsg=

{0} does not have text in the index.
It may have no text, not been analyzed yet, or keyword search was not enabled during ingest.

ExtractedContentViewer.getSolrContent.txtBodyItal={0} HighlightedMatchesSource.getMarkup.noMatchMsg=
There were no keyword hits on this page. 
Keyword could have been in file name. 
Advance to another page for hits or choose Extracted Text to view original text..
+HighlightedMatchesSource.getMarkup.queryFailedMsg=
Failed to retrieve keyword hit results. 
Confirm that Autopsy can connect to the Solr server.
HighlightedMatchesSource.toString=Search Results Installer.reportPortError=Indexing server port {0} is not available. Check if your security software does not block {1} and consider changing {2} in {3} property file in the application user folder. Then try rebooting your system if another process was causing the conflict. Installer.reportStopPortError=Indexing server stop port {0} is not available. Consider changing {1} in {2} property file in the application user folder. @@ -212,11 +213,12 @@ Server.openCore.exception.alreadyOpen.msg=Already an open Core\! Explicitely clo Server.queryNumIdxFiles.exception.msg=Error querying number of indexed files, Server.queryNumIdxChunks.exception.msg=Error querying number of indexed chunks, Server.queryNumIdxDocs.exception.msg=Error querying number of indexed documents, -Server.queryIsIdxd.exception.msg=Error checkign if content is indexed, +Server.queryIsIdxd.exception.msg=Error checking if content is indexed, Server.queryNumFileChunks.exception.msg=Error getting number of file chunks, Server.query.exception.msg=Error running query\: {0} Server.query2.exception.msg=Error running query\: {0} Server.queryTerms.exception.msg=Error running terms query\: {0} +Server.connect.exception.msg=Failed to connect to Solr server\: Server.openCore.exception.msg=Core open requested, but server not yet running Server.openCore.exception.cantOpen.msg=Could not open Core Server.openCore.exception.cantOpen.msg2=Could not open Core diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedTextMarkup.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedTextMarkup.java index c5f5d131a4..069ad0802b 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedTextMarkup.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedTextMarkup.java @@ -34,7 +34,6 @@ import org.apache.solr.client.solrj.response.QueryResponse; import org.sleuthkit.autopsy.coreutils.Version; import org.sleuthkit.autopsy.datamodel.TextMarkupLookup; import org.sleuthkit.autopsy.keywordsearch.KeywordQueryFilter.FilterType; -import org.sleuthkit.datamodel.Content; /** * Highlights hits for a given document. Knows about pages and such for the content viewer. @@ -379,12 +378,8 @@ class HighlightedTextMarkup implements TextMarkup, TextMarkupLookup { return "
" + highlightedContent + "
"; //NON-NLS } - } catch (NoOpenCoreException ex) { - logger.log(Level.WARNING, "Couldn't query markup for page: " + currentPage, ex); //NON-NLS - return ""; - } catch (KeywordSearchModuleException ex) { - logger.log(Level.WARNING, "Could not query markup for page: " + currentPage, ex); //NON-NLS - return ""; + } catch (NoOpenCoreException | KeywordSearchModuleException ex) { + return NbBundle.getMessage(this.getClass(), "HighlightedMatchesSource.getMarkup.queryFailedMsg"); } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java index c9b7a43839..b2804b0017 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java @@ -33,14 +33,15 @@ import org.apache.solr.client.solrj.SolrRequest.METHOD; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; +import org.openide.util.NbBundle; import org.sleuthkit.autopsy.casemodule.Case; import org.sleuthkit.autopsy.coreutils.EscapeUtil; +import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.Version; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE; -import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.SleuthkitCase; import org.sleuthkit.datamodel.TskException; @@ -202,48 +203,51 @@ class LuceneQuery implements KeywordSearchQuery { final Server solrServer = KeywordSearch.getServer(); SolrQuery q = createAndConfigureSolrQuery(snippets); + QueryResponse response; + SolrDocumentList resultList; + Map>> highlightResponse; + Set uniqueSolrDocumentsWithHits; + + try { + response = solrServer.query(q, METHOD.POST); + + resultList = response.getResults(); + + // objectId_chunk -> "text" -> List of previews + highlightResponse = response.getHighlighting(); + + // get the unique set of files with hits + uniqueSolrDocumentsWithHits = filterOneHitPerDocument(resultList); + } + catch (KeywordSearchModuleException ex) { + logger.log(Level.SEVERE, "Error executing Lucene Solr Query: " + keywordString, ex); //NON-NLS + MessageNotifyUtil.Notify.error(NbBundle.getMessage(Server.class, "Server.query.exception.msg", keywordString), ex.getCause().getMessage()); + return matches; + } // cycle through results in sets of MAX_RESULTS for (int start = 0; !allMatchesFetched; start = start + MAX_RESULTS) { q.setStart(start); + allMatchesFetched = start + MAX_RESULTS >= resultList.getNumFound(); + + SleuthkitCase sleuthkitCase; try { - QueryResponse response = solrServer.query(q, METHOD.POST); - SolrDocumentList resultList = response.getResults(); - - // objectId_chunk -> "text" -> List of previews - Map>> highlightResponse = response.getHighlighting(); - - // get the unique set of files with hits - Set uniqueSolrDocumentsWithHits = filterOneHitPerDocument(resultList); - - allMatchesFetched = start + MAX_RESULTS >= resultList.getNumFound(); - - SleuthkitCase sleuthkitCase; - try { - sleuthkitCase = Case.getCurrentCase().getSleuthkitCase(); - } catch (IllegalStateException ex) { - //no case open, must be just closed - return matches; - } - - for (SolrDocument resultDoc : uniqueSolrDocumentsWithHits) { - KeywordHit contentHit; - try { - contentHit = createKeywordtHit(resultDoc, highlightResponse, sleuthkitCase); - } catch (TskException ex) { - return matches; - } - matches.add(contentHit); - } - - } catch (NoOpenCoreException ex) { - logger.log(Level.WARNING, "Error executing Lucene Solr Query: " + keywordString, ex); //NON-NLS - throw ex; - } catch (KeywordSearchModuleException ex) { - logger.log(Level.WARNING, "Error executing Lucene Solr Query: " + keywordString, ex); //NON-NLS + sleuthkitCase = Case.getCurrentCase().getSleuthkitCase(); + } catch (IllegalStateException ex) { + //no case open, must be just closed + return matches; } + for (SolrDocument resultDoc : uniqueSolrDocumentsWithHits) { + KeywordHit contentHit; + try { + contentHit = createKeywordtHit(resultDoc, highlightResponse, sleuthkitCase); + } catch (TskException ex) { + return matches; + } + matches.add(contentHit); + } } return matches; } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java index 49a62021a0..5ff73f4f98 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Server.java @@ -33,7 +33,6 @@ import java.net.ConnectException; import java.net.ServerSocket; import java.net.SocketException; import java.nio.charset.Charset; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -65,6 +64,7 @@ import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrException; import org.sleuthkit.autopsy.casemodule.Case.CaseType; import org.sleuthkit.autopsy.core.UserPreferences; +import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; /** * Handles for keeping track of a Solr server and its cores @@ -665,6 +665,23 @@ public class Server { * @return */ private synchronized Core openCore(Case theCase) throws KeywordSearchModuleException { + try { + if (theCase.getCaseType() == CaseType.SINGLE_USER_CASE) { + currentSolrServer = this.localSolrServer; + } + else { + String host = UserPreferences.getIndexingServerHost(); + String port = UserPreferences.getIndexingServerPort(); + + currentSolrServer = new HttpSolrServer("http://" + host + ":" + port + "/solr"); //NON-NLS + } + connectToSolrServer(currentSolrServer); + } + catch (SolrServerException | IOException ex) { + MessageNotifyUtil.Notify.error(NbBundle.getMessage(Server.class, "Server.connect.exception.msg"), ex.getCause().getMessage()); + throw new KeywordSearchModuleException(NbBundle.getMessage(Server.class, "Server.connect.exception.msg")); + } + String dataDir = getIndexDirPath(theCase); String coreName = theCase.getTextIndexName(); return this.openCore(coreName.isEmpty() ? DEFAULT_CORE_NAME : coreName, new File(dataDir), theCase.getCaseType()); @@ -696,7 +713,7 @@ public class Server { * * @return int representing number of indexed files * @throws KeywordSearchModuleException - * @throws NoOpenCoreExceptionn + * @throws NoOpenCoreException */ public int queryNumIndexedFiles() throws KeywordSearchModuleException, NoOpenCoreException { if (currentCore == null) { @@ -940,6 +957,7 @@ public class Server { * @return new core */ private Core openCore(String coreName, File dataDir, CaseType caseType) throws KeywordSearchModuleException { + try { if (!dataDir.exists()) { dataDir.mkdirs(); @@ -952,14 +970,6 @@ public class Server { NbBundle.getMessage(this.getClass(), "Server.openCore.exception.msg")); } - if (caseType == CaseType.SINGLE_USER_CASE) { - currentSolrServer = this.localSolrServer; - //createCore.setInstanceDir(instanceDir); - } - else { - currentSolrServer = connectToRemoteSolrServer(); - } - if (!isCoreLoaded(coreName)) { CoreAdminRequest.Create createCore = new CoreAdminRequest.Create(); createCore.setDataDir(dataDir.getAbsolutePath()); @@ -984,11 +994,14 @@ public class Server { } } - HttpSolrServer connectToRemoteSolrServer() { - String host = UserPreferences.getIndexingServerHost(); - String port = UserPreferences.getIndexingServerPort(); - - return new HttpSolrServer("http://" + host + ":" + port + "/solr"); + /** + * Attempts to connect to the given Solr server. + * @param solrServer + * @throws SolrServerException + * @throws IOException + */ + void connectToSolrServer(HttpSolrServer solrServer) throws SolrServerException, IOException { + CoreAdminRequest.getStatus(null, solrServer); } /** diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java index 51d2e3ebbe..c047e67009 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SolrSearchService.java @@ -22,15 +22,14 @@ import java.io.IOException; import java.util.HashMap; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.HttpSolrServer; -import org.apache.solr.client.solrj.request.CoreAdminRequest; import org.sleuthkit.datamodel.BlackboardArtifact; import org.sleuthkit.datamodel.BlackboardAttribute; import org.sleuthkit.datamodel.TskCoreException; import org.sleuthkit.autopsy.keywordsearchservice.KeywordSearchService; import org.apache.solr.common.util.ContentStreamBase.StringStream; -import org.openide.util.Exceptions; import org.openide.util.lookup.ServiceProvider; import org.sleuthkit.autopsy.casemodule.Case; +import org.sleuthkit.autopsy.core.UserPreferences; import org.sleuthkit.autopsy.datamodel.ContentUtils; import org.sleuthkit.datamodel.AbstractFile; import org.sleuthkit.datamodel.Content; @@ -152,8 +151,10 @@ public class SolrSearchService implements KeywordSearchService { @Override public boolean canConnectToRemoteSolrServer() { try { - HttpSolrServer solrServer = KeywordSearch.getServer().connectToRemoteSolrServer(); - CoreAdminRequest.getStatus(null, solrServer); + String host = UserPreferences.getIndexingServerHost(); + String port = UserPreferences.getIndexingServerPort(); + HttpSolrServer solrServer = new HttpSolrServer("http://" + host + ":" + port + "/solr"); //NON-NLS; + KeywordSearch.getServer().connectToSolrServer(solrServer); } catch (SolrServerException | IOException ex) { return false;