diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java index dbd3e41f80..a1eba61173 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/HighlightedMatchesSource.java @@ -57,11 +57,11 @@ class HighlightedMatchesSource implements MarkupSource { public String getMarkup() { SolrQuery q = new SolrQuery(); - final String queryEscaped = KeywordSearchUtil.escapeLuceneQuery(solrQuery, true); + final String queryEscaped = KeywordSearchUtil.escapeLuceneQuery(solrQuery, true, false); q.setQuery(queryEscaped); q.addFilterQuery("id:" + content.getId()); - q.addHighlightField("content"); + q.addHighlightField("content"); //for exact highlighting, try content_ws field (with stored="true" in Solr schema) q.setHighlightSimplePre(HIGHLIGHT_PRE); q.setHighlightSimplePost(HIGHLIGHT_POST); q.setHighlightFragsize(0); // don't fragment the highlight diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchResultFactory.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchResultFactory.java index 77494d1640..8be35ba8a0 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchResultFactory.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchResultFactory.java @@ -70,8 +70,7 @@ public class KeywordSearchResultFactory extends ChildFactory { public String toString() { return "Match"; } - }, - } + },} private Presentation presentation; private Collection queries; private Collection things; @@ -149,6 +148,7 @@ public class KeywordSearchResultFactory extends ChildFactory { childFactory = new ResultCollapsedChildFactory(thing); final Node ret = new KeyValueNode(thing, Children.create(childFactory, true)); SwingUtilities.invokeLater(new Runnable() { + @Override public void run() { //DataResultViewerTable view = Utilities.actionsGlobalContext().lookup(DataResultViewerTable.class); @@ -199,11 +199,11 @@ public class KeywordSearchResultFactory extends ChildFactory { final int lastTerm = terms.size() - 1; int curTerm = 0; for (Term term : terms) { - final String termS = KeywordSearchUtil.escapeLuceneQuery(term.getTerm(), true); + final String termS = KeywordSearchUtil.escapeLuceneQuery(term.getTerm(), true, false); if (!termS.contains("*")) { highlightQuery.append(termS); if (lastTerm != curTerm) { - highlightQuery.append(" "); + highlightQuery.append(" "); //acts as OR || } } } @@ -304,25 +304,27 @@ public class KeywordSearchResultFactory extends ChildFactory { final String contentStr = KeywordSearch.getServer().getCore().getSolrContent(content); - //make sure the file contains a match (this gets rid of large number of false positives) - //TODO option in GUI to include approximate matches (faster) - boolean matchFound = false; - if (contentStr != null) {//if not null, some error getting from Solr, handle it by not filtering out - //perform java regex to validate match from Solr - String origQuery = thingContent.getQuery(); + //postprocess + //make sure Solr result contains a match (this gets rid of large number of false positives) + boolean postprocess = true; + boolean matchFound = true; + if (postprocess) { + if (contentStr != null) {//if not null, some error getting from Solr, handle it by not filtering out + //perform java regex to validate match from Solr + String origQuery = thingContent.getQuery(); + + //since query is a match result, we can assume literal pattern + origQuery = Pattern.quote(origQuery); + Pattern p = Pattern.compile(origQuery, Pattern.CASE_INSENSITIVE | Pattern.DOTALL); - //escape the regex query because it may contain special characters from the previous match - //since it's a match result, we can assume literal pattern - origQuery = Pattern.quote(origQuery); - Pattern p = Pattern.compile(origQuery, Pattern.CASE_INSENSITIVE | Pattern.DOTALL); - - Matcher m = p.matcher(contentStr); - matchFound = m.find(); + Matcher m = p.matcher(contentStr); + matchFound = m.find(); + } } if (matchFound) { Node kvNode = new KeyValueNode(thingContent, Children.LEAF); - //wrap in KeywordSearchFilterNode for the markup content, might need to override FilterNode for more customization + //wrap in KeywordSearchFilterNode for the markup content HighlightedMatchesSource highlights = new HighlightedMatchesSource(content, query); return new KeywordSearchFilterNode(highlights, kvNode, query); } else { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchUtil.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchUtil.java index 9b8afa65a1..2940696605 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchUtil.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchUtil.java @@ -16,7 +16,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.sleuthkit.autopsy.keywordsearch; import java.awt.Component; @@ -31,8 +30,10 @@ import org.sleuthkit.datamodel.TskException; public class KeywordSearchUtil { - public enum DIALOG_MESSAGE_TYPE {ERROR, WARN, INFO}; - + public enum DIALOG_MESSAGE_TYPE { + + ERROR, WARN, INFO + }; private static final Logger logger = Logger.getLogger(KeywordSearchUtil.class.getName()); public static String buildDirName(FsContent f) { @@ -65,42 +66,44 @@ public class KeywordSearchUtil { * such as /+-&|!(){}[]^"~*?:\ and treat the whole query as literal word * @return encoded query */ - public static String escapeLuceneQuery(String query, boolean escapeLuceneChars) { + public static String escapeLuceneQuery(String query, boolean escapeLuceneChars, boolean encode) { String queryEscaped = null; String inputString = query; - + if (escapeLuceneChars == true) { final String ESCAPE_CHARS = "/+-&|!(){}[]^\"~*?:\\"; StringBuilder sb = new StringBuilder(); - for (int i = 0; i< inputString.length(); ++i) { + for (int i = 0; i < inputString.length(); ++i) { char c = inputString.charAt(i); - if (ESCAPE_CHARS.contains(Character.toString(c)) ) { + if (ESCAPE_CHARS.contains(Character.toString(c))) { sb.append("\\"); } sb.append(c); } - inputString = sb.toString(); + queryEscaped = inputString = sb.toString(); } - - try { - queryEscaped = URLEncoder.encode(inputString, "UTF-8"); - } - catch (UnsupportedEncodingException ex) { - logger.log(Level.SEVERE, "Error escaping URL query, should not happen.", ex); - queryEscaped = query; + + if (encode) { + try { + queryEscaped = URLEncoder.encode(inputString, "UTF-8"); + } catch (UnsupportedEncodingException ex) { + logger.log(Level.SEVERE, "Error escaping URL query, should not happen.", ex); + queryEscaped = query; + } } return queryEscaped; } - - + public static void displayDialog(final String title, final String message, final DIALOG_MESSAGE_TYPE type) { int messageType; - if (type == DIALOG_MESSAGE_TYPE.ERROR) + if (type == DIALOG_MESSAGE_TYPE.ERROR) { messageType = JOptionPane.ERROR_MESSAGE; - else if (type == DIALOG_MESSAGE_TYPE.WARN) + } else if (type == DIALOG_MESSAGE_TYPE.WARN) { messageType = JOptionPane.WARNING_MESSAGE; - else messageType = JOptionPane.INFORMATION_MESSAGE; - + } else { + messageType = JOptionPane.INFORMATION_MESSAGE; + } + final Component parentComponent = null; // Use default window frame. JOptionPane.showMessageDialog( parentComponent, @@ -108,4 +111,20 @@ public class KeywordSearchUtil { title, messageType); } + + public static boolean displayConfirmDialog(final String title, final String message, final DIALOG_MESSAGE_TYPE type) { + int messageType; + if (type == DIALOG_MESSAGE_TYPE.ERROR) { + messageType = JOptionPane.ERROR_MESSAGE; + } else if (type == DIALOG_MESSAGE_TYPE.WARN) { + messageType = JOptionPane.WARNING_MESSAGE; + } else { + messageType = JOptionPane.INFORMATION_MESSAGE; + } + if (JOptionPane.showConfirmDialog(null, message, title, JOptionPane.YES_NO_OPTION, messageType) == JOptionPane.YES_OPTION) { + return true; + } else { + return false; + } + } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java index 82d3799523..29e199b9e4 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java @@ -56,7 +56,7 @@ public class LuceneQuery implements KeywordSearchQuery { @Override public void escape() { - queryEscaped = KeywordSearchUtil.escapeLuceneQuery(query, true); + queryEscaped = KeywordSearchUtil.escapeLuceneQuery(query, true, true); isEscaped = true; } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java index db9f995bdd..83492307d0 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java @@ -49,7 +49,7 @@ import org.sleuthkit.autopsy.keywordsearch.KeywordSearchQueryManager.Presentatio import org.sleuthkit.datamodel.FsContent; public class TermComponentQuery implements KeywordSearchQuery { - + private static final int TERMS_UNLIMITED = -1; //corresponds to field in Solr schema, analyzed with white-space tokenizer only private static final String TERMS_SEARCH_FIELD = "content_ws"; @@ -60,14 +60,14 @@ public class TermComponentQuery implements KeywordSearchQuery { private String queryEscaped; private boolean isEscaped; private List terms; - + public TermComponentQuery(String query) { this.termsQuery = query; this.queryEscaped = query; isEscaped = false; terms = null; } - + @Override public void escape() { //treat as literal @@ -77,13 +77,13 @@ public class TermComponentQuery implements KeywordSearchQuery { queryEscaped = Pattern.quote(termsQuery); isEscaped = true; } - + @Override public boolean validate() { if (queryEscaped.equals("")) { return false; } - + boolean valid = true; try { Pattern.compile(queryEscaped); @@ -110,9 +110,9 @@ public class TermComponentQuery implements KeywordSearchQuery { q.setTermsRegex(queryEscaped); q.addTermsField(TERMS_SEARCH_FIELD); q.setTimeAllowed(TERMS_TIMEOUT); - + return q; - + } /* @@ -120,7 +120,7 @@ public class TermComponentQuery implements KeywordSearchQuery { */ protected List executeQuery(SolrQuery q) { Server.Core solrCore = KeywordSearch.getServer().getCore(); - + List termsCol = null; try { TermsResponse tr = solrCore.queryTerms(q); @@ -131,17 +131,17 @@ public class TermComponentQuery implements KeywordSearchQuery { return null; //no need to create result view, just display error dialog } } - + @Override public String getEscapedQueryString() { return this.queryEscaped; } - + @Override public String getQueryString() { return this.termsQuery; } - + @Override public Collection getTerms() { return terms; @@ -154,7 +154,7 @@ public class TermComponentQuery implements KeywordSearchQuery { @Override public List performQuery() { List results = new ArrayList(); - + final SolrQuery q = createQuery(); terms = executeQuery(q); @@ -168,20 +168,21 @@ public class TermComponentQuery implements KeywordSearchQuery { final int lastTerm = terms.size() - 1; int curTerm = 0; for (Term term : terms) { - final String termS = term.getTerm(); + final String termS = KeywordSearchUtil.escapeLuceneQuery(term.getTerm(), true, false); + //final String termS = term.getTerm(); if (!termS.contains("*")) { - filesQueryB.append(termS); + filesQueryB.append(TERMS_SEARCH_FIELD).append(":").append(termS); if (curTerm != lastTerm) { - filesQueryB.append(" "); + filesQueryB.append(" "); //acts as OR || } } ++curTerm; } List uniqueMatches = new ArrayList(); - + if (!terms.isEmpty()) { LuceneQuery filesQuery = new LuceneQuery(filesQueryB.toString()); - filesQuery.escape(); + //filesQuery.escape(); try { uniqueMatches = filesQuery.performQuery(); } catch (RuntimeException e) { @@ -190,28 +191,33 @@ public class TermComponentQuery implements KeywordSearchQuery { } + //result postprocessing //filter out non-matching files using the original query (whether literal or not) - //TODO this could be costly, for now just testing how it performs - for (FsContent f : uniqueMatches) { - Pattern p = Pattern.compile(queryEscaped, Pattern.CASE_INSENSITIVE | Pattern.DOTALL); - final String contentStr = KeywordSearch.getServer().getCore().getSolrContent(f); - Matcher m = p.matcher(contentStr); - if (m.find()) { - results.add(f); + boolean postprocess = false; + if (postprocess) { + for (FsContent f : uniqueMatches) { + Pattern p = Pattern.compile(queryEscaped, Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + final String contentStr = KeywordSearch.getServer().getCore().getSolrContent(f); + Matcher m = p.matcher(contentStr); + if (m.find()) { + results.add(f); + } } + } else { + results.addAll(uniqueMatches); } - - - + + + return results; } - + @Override public void execute() { SolrQuery q = createQuery(); - + logger.log(Level.INFO, "Executing TermsComponent query: " + q.toString()); - + final SwingWorker worker = new TermsQueryWorker(q); worker.execute(); } @@ -221,9 +227,9 @@ public class TermComponentQuery implements KeywordSearchQuery { * @param terms */ private void publishNodes(List terms) { - + Collection things = new ArrayList(); - + Iterator it = terms.iterator(); int termID = 0; //long totalMatches = 0; @@ -237,17 +243,17 @@ public class TermComponentQuery implements KeywordSearchQuery { things.add(new KeyValueThing(match, kvs, ++termID)); //totalMatches += matches; } - + Node rootNode = null; if (things.size() > 0) { Children childThingNodes = Children.create(new KeywordSearchResultFactory(termsQuery, things, Presentation.DETAIL), true); - + rootNode = new AbstractNode(childThingNodes); } else { rootNode = Node.EMPTY; } - + final String pathText = "Term query"; // String pathText = "RegEx query: " + termsQuery //+ " Files with exact matches: " + Long.toString(totalMatches) + " (also listing approximate matches)"; @@ -256,29 +262,29 @@ public class TermComponentQuery implements KeywordSearchQuery { searchResultWin.requestActive(); // make it the active top component } - + class TermsQueryWorker extends SwingWorker, Void> { - + private SolrQuery q; private ProgressHandle progress; - + TermsQueryWorker(SolrQuery q) { this.q = q; } - + @Override protected List doInBackground() throws Exception { progress = ProgressHandleFactory.createHandle("Terms query task"); progress.start(); progress.progress("Running Terms query."); - + terms = executeQuery(q); - + progress.progress("Terms query completed."); - + return terms; } - + @Override protected void done() { if (!this.isCancelled()) { @@ -287,7 +293,7 @@ public class TermComponentQuery implements KeywordSearchQuery { publishNodes(terms); } catch (InterruptedException e) { logger.log(Level.INFO, "Exception while executing regex query,", e); - + } catch (ExecutionException e) { logger.log(Level.INFO, "Exception while executing regex query,", e); } finally {