From 40b3a52f92b2515fe72df72f0230a4290081fa0e Mon Sep 17 00:00:00 2001 From: millmanorama Date: Wed, 8 Mar 2017 14:14:23 +0100 Subject: [PATCH 1/6] save hits even if we never get a snippet --- .../autopsy/keywordsearch/QueryResults.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java index 8d47831dba..eeb7b6dfaa 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java @@ -149,16 +149,14 @@ class QueryResults { continue; } } - if (snippet != null) { - KeywordCachedArtifact writeResult = keywordSearchQuery.writeSingleFileHitsToBlackBoard(keyword, hit, snippet, keywordList.getName()); - if (writeResult != null) { - newArtifacts.add(writeResult.getArtifact()); - if (notifyInbox) { - writeSingleFileInboxMessage(writeResult, hit.getContent()); - } - } else { - logger.log(Level.WARNING, "BB artifact for keyword hit not written, file: {0}, hit: {1}", new Object[]{hit.getContent(), keyword.toString()}); //NON-NLS + KeywordCachedArtifact writeResult = keywordSearchQuery.writeSingleFileHitsToBlackBoard(keyword, hit, snippet, keywordList.getName()); + if (writeResult != null) { + newArtifacts.add(writeResult.getArtifact()); + if (notifyInbox) { + writeSingleFileInboxMessage(writeResult, hit.getContent()); } + } else { + logger.log(Level.WARNING, "BB artifact for keyword hit not written, file: {0}, hit: {1}", new Object[]{hit.getContent(), keyword.toString()}); //NON-NLS } } ++unitProgress; From 6fa88eb1327b83fc2335e2a6323631f0c24c93b8 Mon Sep 17 00:00:00 2001 From: millmanorama Date: Thu, 9 Mar 2017 10:51:25 +0100 Subject: [PATCH 2/6] improve comment about snippet queries --- .../org/sleuthkit/autopsy/keywordsearch/QueryResults.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java index eeb7b6dfaa..084e823bd3 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java @@ -138,7 +138,11 @@ class QueryResults { if (StringUtils.isBlank(snippet)) { final String snippetQuery = KeywordSearchUtil.escapeLuceneQuery(termString); try { - //this doesn't work for regex queries... + /* + * this doesn't work for regex queries... But that is + * okay because regex queries always have snippets made + * from the content_str field we pull back from Solr + */ snippet = LuceneQuery.querySnippet(snippetQuery, hit.getSolrObjectId(), hit.getChunkId(), !keywordSearchQuery.isLiteral(), true); } catch (NoOpenCoreException e) { logger.log(Level.WARNING, "Error querying snippet: " + snippetQuery, e); //NON-NLS From d676148af86dc0cd1be5d912f52ffadc3a9c6440 Mon Sep 17 00:00:00 2001 From: millmanorama Date: Thu, 9 Mar 2017 11:08:58 +0100 Subject: [PATCH 3/6] remove unnecessary field and constructor parameter --- .../autopsy/keywordsearch/LuceneQuery.java | 2 +- .../autopsy/keywordsearch/QueryResults.java | 18 +++++------------- .../autopsy/keywordsearch/RegexQuery.java | 2 +- .../autopsy/keywordsearch/SearchRunner.java | 2 +- .../keywordsearch/TermsComponentQuery.java | 2 +- 5 files changed, 9 insertions(+), 17 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java index 789f238329..7d98d3c3ff 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java @@ -120,7 +120,7 @@ class LuceneQuery implements KeywordSearchQuery { @Override public QueryResults performQuery() throws KeywordSearchModuleException, NoOpenCoreException { - QueryResults results = new QueryResults(this, keywordList); + QueryResults results = new QueryResults(this); //in case of single term literal query there is only 1 term boolean showSnippets = KeywordSearchSettings.getShowSnippets(); results.addResult(new Keyword(keywordString, true), performLuceneQuery(showSnippets)); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java index 084e823bd3..410ec1a94c 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/QueryResults.java @@ -60,25 +60,17 @@ class QueryResults { */ private final Map> results = new HashMap<>(); - /** - * The list of keywords - */ - // TODO: This is redundant. The keyword list is in the query. - private final KeywordList keywordList; + - QueryResults(KeywordSearchQuery query, KeywordList keywordList) { + QueryResults(KeywordSearchQuery query) { this.keywordSearchQuery = query; - this.keywordList = keywordList; } void addResult(Keyword keyword, List hits) { results.put(keyword, hits); } - // TODO: This is redundant. The keyword list is in the query. - KeywordList getKeywordList() { - return keywordList; - } + KeywordSearchQuery getQuery() { return keywordSearchQuery; @@ -129,7 +121,7 @@ class QueryResults { if (hitDisplayStr.length() > 50) { hitDisplayStr = hitDisplayStr.substring(0, 49) + "..."; } - subProgress.progress(keywordList.getName() + ": " + hitDisplayStr, unitProgress); + subProgress.progress(keywordSearchQuery.getKeywordList().getName() + ": " + hitDisplayStr, unitProgress); } for (KeywordHit hit : getOneHitPerObject(keyword)) { @@ -153,7 +145,7 @@ class QueryResults { continue; } } - KeywordCachedArtifact writeResult = keywordSearchQuery.writeSingleFileHitsToBlackBoard(keyword, hit, snippet, keywordList.getName()); + KeywordCachedArtifact writeResult = keywordSearchQuery.writeSingleFileHitsToBlackBoard(keyword, hit, snippet, keywordSearchQuery.getKeywordList().getName()); if (writeResult != null) { newArtifacts.add(writeResult.getArtifact()); if (notifyInbox) { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java index ea5e330ca9..fcf0881272 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/RegexQuery.java @@ -154,7 +154,7 @@ final class RegexQuery implements KeywordSearchQuery { @Override public QueryResults performQuery() throws NoOpenCoreException { - QueryResults results = new QueryResults(this, keywordList); + QueryResults results = new QueryResults(this); final Server solrServer = KeywordSearch.getServer(); SolrQuery solrQuery = new SolrQuery(); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java index 2090aa3187..ce0d07bc78 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/SearchRunner.java @@ -590,7 +590,7 @@ public final class SearchRunner { // Create a new (empty) QueryResults object to hold the most recently // found hits. - QueryResults newResults = new QueryResults(queryResult.getQuery(), queryResult.getKeywordList()); + QueryResults newResults = new QueryResults(queryResult.getQuery()); // For each keyword represented in the results. for (Keyword keyword : queryResult.getKeywords()) { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsComponentQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsComponentQuery.java index 44cc8ab5e9..7ecfdbe35a 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsComponentQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermsComponentQuery.java @@ -280,7 +280,7 @@ final class TermsComponentQuery implements KeywordSearchQuery { /* * Do a term query for each term that matched the regex. */ - QueryResults results = new QueryResults(this, keywordList); + QueryResults results = new QueryResults(this); for (Term term : terms) { /* * If searching for credit card account numbers, do a Luhn check on From 2ce24a4a8cb84a5b7376d162a4b8676f486fa3b5 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 9 Mar 2017 09:56:05 -0500 Subject: [PATCH 4/6] Resolve IDE code hints in keywordsearch/HighlightedText.java --- .../autopsy/keywordsearch/HighlightedText.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java index 83f2179132..1549227225 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java @@ -27,7 +27,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; -import java.util.TreeSet; import java.util.logging.Level; import java.util.stream.Collectors; import javax.annotation.concurrent.GuardedBy; @@ -387,7 +386,7 @@ class HighlightedText implements IndexedText { highlightedContent = insertAnchors(highlightedContent); return "
" + highlightedContent + "
"; //NON-NLS - } catch (Exception ex) { + } catch (TskCoreException | KeywordSearchModuleException | NoOpenCoreException ex) { logger.log(Level.SEVERE, "Error getting highlighted text for " + objectId, ex); //NON-NLS return NbBundle.getMessage(this.getClass(), "HighlightedMatchesSource.getMarkup.queryFailedMsg"); } @@ -453,8 +452,8 @@ class HighlightedText implements IndexedText { //we also need to escape the keyword so that it matches the escpared text final String escapedKeyword = StringEscapeUtils.escapeHtml(keyword); int textOffset = 0; - int hitOffset; - while ((hitOffset = StringUtils.indexOfIgnoreCase(text, escapedKeyword, textOffset)) != -1) { + int hitOffset = StringUtils.indexOfIgnoreCase(text, escapedKeyword, textOffset); + while (hitOffset != -1) { // Append the portion of text up to (but not including) the hit. highlightedText.append(text.substring(textOffset, hitOffset)); // Add in the highlighting around the keyword. @@ -464,6 +463,8 @@ class HighlightedText implements IndexedText { // Advance the text offset past the keyword. textOffset = hitOffset + escapedKeyword.length(); + + hitOffset = StringUtils.indexOfIgnoreCase(text, escapedKeyword, textOffset); } // Append the remainder of text field highlightedText.append(text.substring(textOffset, text.length())); @@ -487,22 +488,21 @@ class HighlightedText implements IndexedText { * @return */ private String insertAnchors(String searchableContent) { - int searchOffset = 0; - int index = -1; - StringBuilder buf = new StringBuilder(searchableContent); - final String searchToken = HIGHLIGHT_PRE; final int indexSearchTokLen = searchToken.length(); final String insertPre = ""; //NON-NLS int count = 0; - while ((index = buf.indexOf(searchToken, searchOffset)) >= 0) { + int searchOffset = 0; + int index = buf.indexOf(searchToken, searchOffset); + while (index >= 0) { String insertString = insertPre + Integer.toString(count + 1) + insertPost; int insertStringLen = insertString.length(); buf.insert(index, insertString); searchOffset = index + indexSearchTokLen + insertStringLen; //next offset past this anchor ++count; + index = buf.indexOf(searchToken, searchOffset); } //store total hits for this page, now that we know it From 681f2da55d2515b98a33f8db53295960fffbaa4a Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 9 Mar 2017 10:06:31 -0500 Subject: [PATCH 5/6] Format keywordsearch/HighlightedText.java --- .../src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java | 1 - 1 file changed, 1 deletion(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java index 1549227225..911914426a 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java @@ -243,7 +243,6 @@ class HighlightedText implements IndexedText { @Override public boolean hasNextPage() { return getIndexOfCurrentPage() < pages.size() - 1; - } @Override From 3d4d83e2392b00aa6c753d9a130016fa8dde29e9 Mon Sep 17 00:00:00 2001 From: Richard Cordovano Date: Thu, 9 Mar 2017 10:19:09 -0500 Subject: [PATCH 6/6] Format keywordsearch/HighlightedText.java --- .../autopsy/keywordsearch/HighlightedText.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java index 911914426a..9aa8b74a4f 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedText.java @@ -253,7 +253,7 @@ class HighlightedText implements IndexedText { @Override public int nextPage() { if (hasNextPage()) { - currentPage = Iterators.get(pages.iterator(),getIndexOfCurrentPage() + 1); + currentPage = Iterators.get(pages.iterator(), getIndexOfCurrentPage() + 1); return currentPage; } else { throw new IllegalStateException("No next page."); @@ -263,7 +263,7 @@ class HighlightedText implements IndexedText { @Override public int previousPage() { if (hasPreviousPage()) { - currentPage = Iterators.get(pages.iterator(),getIndexOfCurrentPage()-1); + currentPage = Iterators.get(pages.iterator(), getIndexOfCurrentPage() - 1); return currentPage; } else { throw new IllegalStateException("No previous page."); @@ -462,15 +462,15 @@ class HighlightedText implements IndexedText { // Advance the text offset past the keyword. textOffset = hitOffset + escapedKeyword.length(); - + hitOffset = StringUtils.indexOfIgnoreCase(text, escapedKeyword, textOffset); } // Append the remainder of text field highlightedText.append(text.substring(textOffset, text.length())); - + if (highlightedText.length() == 0) { return NbBundle.getMessage(HighlightedText.class, "HighlightedMatchesSource.getMarkup.noMatchMsg"); - } + } //reset for next pass text = highlightedText.toString(); highlightedText = new StringBuilder(""); @@ -494,7 +494,7 @@ class HighlightedText implements IndexedText { final String insertPost = "'>"; //NON-NLS int count = 0; int searchOffset = 0; - int index = buf.indexOf(searchToken, searchOffset); + int index = buf.indexOf(searchToken, searchOffset); while (index >= 0) { String insertString = insertPre + Integer.toString(count + 1) + insertPost; int insertStringLen = insertString.length();