added query manager and child factories supporting single/multiple, word/regex queries, and 2 ways of presenting nodes (detailed view with all hits, and collapsed view)

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchDataExplorer.java

@@ -79,12 +79,18 @@ public class KeywordSearchDataExplorer implements DataExplorer {
      * @param solrQuery 
      */
     private void search(String query, QueryType queryType) {
-        KeywordSearchQueryManager man = new KeywordSearchQueryManager(query, queryType, Presentation.DETAIL);
+        //TODO populate map with queries for keyword list search here
+        
+        //Map<String, Boolean>qmap = new LinkedHashMap<String, Boolean>();
+        //qmap.put(query, Boolean.FALSE);
+        //KeywordSearchQueryManager man = new KeywordSearchQueryManager(query, queryType, Presentation.COLLAPSE);
+        
+        KeywordSearchQueryManager man = new KeywordSearchQueryManager(query, queryType, Presentation.COLLAPSE);
 
         if (man.validate()) {
             man.execute();
         } else {
-            displayErrorDialog("Invalid query syntax: " + query);
+            displayErrorDialog("Invalid query syntax." );
         }
 
     }

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQuery.java

@@ -18,7 +18,9 @@
  */
 package org.sleuthkit.autopsy.keywordsearch;
 
+import java.util.Collection;
 import java.util.List;
+import org.apache.solr.client.solrj.response.TermsResponse.Term;
 import org.sleuthkit.datamodel.FsContent;
 
 public interface KeywordSearchQuery {
@@ -59,5 +61,11 @@ public interface KeywordSearchQuery {
      */
     public String getEscapedQueryString();
     
+    /**
+     * get terms associated with the query if any
+     * @return collection of terms associated with the query
+     */
+    public Collection<Term>getTerms();
+    
     
 }

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQueryManager.java

@@ -19,11 +19,19 @@
 package org.sleuthkit.autopsy.keywordsearch;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import org.apache.solr.client.solrj.response.TermsResponse.Term;
+import org.openide.nodes.AbstractNode;
+import org.openide.nodes.Children;
+import org.openide.nodes.Node;
+import org.openide.windows.TopComponent;
+import org.sleuthkit.autopsy.corecomponents.DataResultTopComponent;
+import org.sleuthkit.autopsy.datamodel.KeyValueThing;
 import org.sleuthkit.autopsy.keywordsearch.KeywordSearch.QueryType;
 import org.sleuthkit.datamodel.FsContent;
 
@@ -81,29 +89,45 @@ public class KeywordSearchQueryManager implements KeywordSearchQuery {
                 default:
                     ;
             }
-
-            escape();
             queryDelegates.add(del);
+
         }
+        escape();
 
     }
 
     @Override
     public void execute() {
-
+        //execute and present the query
+        //delegate query to query objects and presentation child factories
         if (queryType == QueryType.WORD || presentation == Presentation.DETAIL) {
             for (KeywordSearchQuery q : queryDelegates) {
                 q.execute();
             }
         } else {
+            //Collapsed view
+            Collection<KeyValueThing> things = new ArrayList<KeyValueThing>();
+            int queryID = 0;
             for (KeywordSearchQuery q : queryDelegates) {
-                List<FsContent> fsContents = q.performQuery();
+                Map<String, Object> kvs = new LinkedHashMap<String, Object>();
-                //TODO create view, send to proper factory
+                final String queryStr = q.getQueryString();
+                things.add(new KeyValueThingQuery(queryStr, kvs, ++queryID, q));
             }
 
+            Node rootNode = null;
+            if (things.size() > 0) {
+                Children childThingNodes =
+                        Children.create(new KeywordSearchResultFactory(queries.keySet(), things, Presentation.COLLAPSE), true);
+
+                rootNode = new AbstractNode(childThingNodes);
+            } else {
+                rootNode = Node.EMPTY;
             }
 
-
+            final String pathText = "Keyword query";
+            TopComponent searchResultWin = DataResultTopComponent.createInstance("Keyword search", pathText, rootNode, things.size());
+            searchResultWin.requestActive();
+        }
     }
 
     @Override
@@ -123,6 +147,19 @@ public class KeywordSearchQueryManager implements KeywordSearchQuery {
         return null;
     }
 
+    @Override
+    public boolean validate() {
+        boolean allValid = true;
+        for (KeywordSearchQuery tcq : queryDelegates) {
+            if (!tcq.validate()) {
+                logger.log(Level.WARNING, "Query has invalid syntax: " + tcq.getQueryString());
+                allValid = false;
+                break;
+            }
+        }
+        return allValid;
+    }
+
     @Override
     public String getEscapedQueryString() {
         StringBuilder sb = new StringBuilder();
@@ -144,15 +181,24 @@ public class KeywordSearchQueryManager implements KeywordSearchQuery {
     }
 
     @Override
-    public boolean validate() {
+    public Collection<Term> getTerms() {
-        boolean allValid = true;
+        return null;
-        for (KeywordSearchQuery tcq : queryDelegates) {
+    }
-            if (!tcq.validate()) {
+}
-                logger.log(Level.WARNING, "Query has invalid syntax: " + tcq.getQueryString());
+
-                allValid = false;
+/*
-                break;
+ * custom KeyValueThing that also stores query object  to execute
-            }
+ */
-        }
+class KeyValueThingQuery extends KeyValueThing {
-        return allValid;
+
+    private KeywordSearchQuery query;
+
+    KeywordSearchQuery getQuery() {
+        return query;
+    }
+
+    public KeyValueThingQuery(String name, Map<String, Object> map, int id, KeywordSearchQuery query) {
+        super(name, map, id);
+        this.query = query;
     }
 }

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchResultFactory.java

@@ -21,13 +21,14 @@ package org.sleuthkit.autopsy.keywordsearch;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.TreeSet;
 import java.util.logging.Logger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import org.apache.solr.client.solrj.response.TermsResponse.Term;
 import org.openide.nodes.ChildFactory;
 import org.openide.nodes.Children;
 import org.openide.nodes.Node;
@@ -114,12 +115,24 @@ public class KeywordSearchResultFactory extends ChildFactory<KeyValueThing> {
     @Override
     protected boolean createKeys(List<KeyValueThing> toPopulate) {
         int id = 0;
+        if (presentation == Presentation.DETAIL) {
             for (String query : queries) {
                 Map<String, Object> map = new LinkedHashMap<String, Object>();
                 initCommonProperties(map);
                 setCommonProperty(map, CommonPropertyTypes.QUERY, query);
                 toPopulate.add(new KeyValueThing(query, map, ++id));
             }
+        } else {
+            for (KeyValueThing thing : things) {
+                //Map<String, Object> map = new LinkedHashMap<String, Object>();
+                Map<String, Object> map = thing.getMap();
+                initCommonProperties(map);
+                final String query = thing.getName();
+                setCommonProperty(map, CommonPropertyTypes.QUERY, query);
+                //toPopulate.add(new KeyValueThing(query, map, ++id));
+                toPopulate.add(thing);
+            }
+        }
 
         return true;
     }
@@ -129,7 +142,7 @@ public class KeywordSearchResultFactory extends ChildFactory<KeyValueThing> {
         ChildFactory<KeyValueThing> childFactory = null;
         switch (presentation) {
             case COLLAPSE:
-                childFactory = null;
+                childFactory = new ResultCollapsedChildFactory(thing);
                 break;
             case DETAIL:
                 childFactory = new ResulTermsMatchesChildFactory(things);
@@ -145,36 +158,63 @@ public class KeywordSearchResultFactory extends ChildFactory<KeyValueThing> {
      * the node produced is a child node
      * The factory actually executes query.
      */
-    class ResulCollapsedChildFactory extends ChildFactory<KeyValueThing> {
+    class ResultCollapsedChildFactory extends ChildFactory<KeyValueThing> {
 
         KeyValueThing queryThing;
 
-        ResulCollapsedChildFactory(KeyValueThing queryThing) {
+        ResultCollapsedChildFactory(KeyValueThing queryThing) {
             this.queryThing = queryThing;
         }
 
         @Override
         protected boolean createKeys(List<KeyValueThing> toPopulate) {
-            String origQuery = queryThing.getName();
+            final String origQuery = queryThing.getName();
-            TermComponentQuery tcq = new TermComponentQuery(origQuery);
+            final KeyValueThingQuery queryThingQuery = (KeyValueThingQuery) queryThing;
-            Map<String,Object> map = new LinkedHashMap<String,Object>();
+            final KeywordSearchQuery tcq = queryThingQuery.getQuery();
-            if (tcq.validate()) {
+
-                map.put("query_valid", true);
+            if (!tcq.validate()) {
-                return true;
+                //TODO mark the particular query node RED
-            }
-            else {
-                map.put("query_valid", false);
                 return false;
             }
 
+            //execute the query and get fscontents matching
+            List<FsContent> fsContents = tcq.performQuery();
 
-            //return toPopulate.addAll(things);
+            //construct a Solr query using aggregated terms to get highlighting
+            //the query is executed later on demand
+            StringBuilder highlightQuery = new StringBuilder();
+            Collection<Term> terms = tcq.getTerms();
+            for (Term term : terms) {
+                final String termS = KeywordSearchUtil.escapeLuceneQuery(term.getTerm());
+                highlightQuery.append(termS);
+                highlightQuery.append(" ");
+            }
+
+            int resID = 0;
+            for (FsContent f : fsContents) {
+                //get unique match result files
+                Map<String, Object> resMap = new LinkedHashMap<String, Object>();
+                AbstractFsContentNode.fillPropertyMap(resMap, f);
+                setCommonProperty(resMap, CommonPropertyTypes.MATCH, f.getName());
+                toPopulate.add(new KeyValueThingContent(f.getName(), resMap, ++resID, f, highlightQuery.toString()));
+            }
+
+            return true;
         }
 
         @Override
         protected Node createNodeForKey(KeyValueThing thing) {
-            return new KeyValueNode(thing, Children.LEAF);
+            //return new KeyValueNode(thing, Children.LEAF);
             //return new KeyValueNode(thing, Children.create(new ResultFilesChildFactory(thing), true));
+            final KeyValueThingContent thingContent = (KeyValueThingContent) thing;
+            final Content content = thingContent.getContent();
+            final String query = thingContent.getQuery();
+
+            Node kvNode = new KeyValueNode(thingContent, Children.LEAF);
+            //wrap in KeywordSearchFilterNode for the markup content, might need to override FilterNode for more customization
+            HighlightedMatchesSource highlights = new HighlightedMatchesSource(content, query);
+            return new KeywordSearchFilterNode(highlights, kvNode, query);
+
         }
     }
 
@@ -223,7 +263,7 @@ public class KeywordSearchResultFactory extends ChildFactory<KeyValueThing> {
                 List<FsContent> matches = filesQuery.performQuery();
 
                 //get unique match result files
-                Set<FsContent> uniqueMatches = new TreeSet<FsContent>();
+                Set<FsContent> uniqueMatches = new LinkedHashSet<FsContent>();
                 uniqueMatches.addAll(matches);
 
                 int resID = 0;
@@ -269,8 +309,7 @@ public class KeywordSearchResultFactory extends ChildFactory<KeyValueThing> {
                     return null;
                 }
             }
-
+        }
-            
     }
 
     /*
@@ -295,5 +334,4 @@ public class KeywordSearchResultFactory extends ChildFactory<KeyValueThing> {
             this.query = query;
         }
     }
-    }
 }

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java

@@ -18,17 +18,17 @@
  */
 package org.sleuthkit.autopsy.keywordsearch;
 
-import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.TermsResponse.Term;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
 import org.openide.nodes.Node;
@@ -68,6 +68,11 @@ public class LuceneQuery implements KeywordSearchQuery {
         return this.query;
     }
     
+    @Override
+    public Collection<Term>getTerms() {
+        return null;
+    }
+
     /**
      * Just perform the query and return result without updating the GUI
      * This utility is used in this class, can be potentially reused by other classes
@@ -129,7 +134,8 @@ public class LuceneQuery implements KeywordSearchQuery {
         escape();
         List<FsContent> matches = performQuery();
 
-        String pathText = "Lucene query: " + query;
+        String pathText = "Keyword query: " + query;
+        
         Node rootNode = new KeywordSearchNode(matches, query);
         Node filteredRootNode = new TableFilterNode(rootNode, true);
         

KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java

@@ -20,12 +20,12 @@ package org.sleuthkit.autopsy.keywordsearch;
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.TreeSet;
 import java.util.concurrent.ExecutionException;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -59,25 +59,34 @@ public class TermComponentQuery implements KeywordSearchQuery {
     private String termsQuery;
     private String queryEscaped;
     private boolean isEscaped;
+    private List<Term> terms;
 
     public TermComponentQuery(String query) {
         this.termsQuery = query;
         this.queryEscaped = query;
         isEscaped = false;
+        terms = null;
     }
 
     @Override
     public void escape() {
         //treat as literal
+        //TODO for actual literal query to work in Java/Solr
+        //might need to either: use terms prefix (not regex) query with the literal
+        //or append .* to the literal regex
         queryEscaped = Pattern.quote(termsQuery);
         isEscaped = true;
     }
 
     @Override
     public boolean validate() {
+        if (queryEscaped.equals("")) {
+            return false;
+        }
+
         boolean valid = true;
         try {
-            Pattern.compile(termsQuery);
+            Pattern.compile(queryEscaped);
         } catch (PatternSyntaxException ex1) {
             valid = false;
         } catch (IllegalArgumentException ex2) {
@@ -98,7 +107,7 @@ public class TermComponentQuery implements KeywordSearchQuery {
         //q.setTermsLimit(200);
         //q.setTermsRegexFlag(regexFlag);
         //q.setTermsRaw(true);
-        q.setTermsRegex(termsQuery);
+        q.setTermsRegex(queryEscaped);
         q.addTermsField(TERMS_SEARCH_FIELD);
         q.setTimeAllowed(TERMS_TIMEOUT);
 
@@ -107,24 +116,22 @@ public class TermComponentQuery implements KeywordSearchQuery {
     }
 
     /*
-     * execute query and return terms
+     * execute query and return terms, helper method
-     * helper method, can be called from the same or threaded query context
      */
     protected List<Term> executeQuery(SolrQuery q) {
         Server.Core solrCore = KeywordSearch.getServer().getCore();
 
-        List<Term> terms = null;
+        List<Term> termsCol = null;
         try {
             TermsResponse tr = solrCore.queryTerms(q);
-            terms = tr.getTerms(TERMS_SEARCH_FIELD);
+            termsCol = tr.getTerms(TERMS_SEARCH_FIELD);
-            return terms;
+            return termsCol;
         } catch (SolrServerException ex) {
             logger.log(Level.SEVERE, "Error executing the regex terms query: " + termsQuery, ex);
             return null;  //no need to create result view, just display error dialog
         }
     }
 
-    
     @Override
     public String getEscapedQueryString() {
         return this.queryEscaped;
@@ -135,6 +142,11 @@ public class TermComponentQuery implements KeywordSearchQuery {
         return this.termsQuery;
     }
 
+    @Override
+    public Collection<Term> getTerms() {
+        return terms;
+    }
+
     /**
      * return collapsed matches with all files for the query
      * without per match breakdown
@@ -144,29 +156,40 @@ public class TermComponentQuery implements KeywordSearchQuery {
         List<FsContent> results = new ArrayList();
 
         final SolrQuery q = createQuery();
-        List<Term> terms = executeQuery(q);
+        terms = executeQuery(q);
-        //get unique match result files
-        Set<FsContent> uniqueMatches = new TreeSet<FsContent>();
         
+        //get unique match result files
+   
+        //execute per term Solr query to get files
+        //more costly
+        /*
+        Set<FsContent> uniqueMatches = new HashSet<FsContent>();
         for (Term term : terms) {
         String word = term.getTerm();
         LuceneQuery filesQuery = new LuceneQuery(word);
         filesQuery.escape();
         List<FsContent> matches = filesQuery.performQuery();
         uniqueMatches.addAll(matches);
-        }
+        }*/
 
-        //filter out non-matching files
+        
-        //escape regex if needed 
+        //combine the terms into single Solr query to get files
-        //(if the original query was not literal, this must be)
+        //TODO limited by GET length limit, try POST ?
-        String literalQuery = null;
+        StringBuilder filesQueryB = new StringBuilder();
-        if (isEscaped) {
+        for (Term term : terms) {
-            literalQuery = this.queryEscaped;
+            final String termS = KeywordSearchUtil.escapeLuceneQuery(term.getTerm());
-        } else {
+            filesQueryB.append(termS);
-            literalQuery = Pattern.quote(this.termsQuery);
+            filesQueryB.append(" ");
         }
+        LuceneQuery filesQuery = new LuceneQuery(filesQueryB.toString());
+        filesQuery.escape();
+        List<FsContent> uniqueMatches = filesQuery.performQuery();
+
+        
+        //filter out non-matching files using the original query (whether literal or not)
+        //TODO this could be costly, for now just testing how it performs
         for (FsContent f : uniqueMatches) {
-            Pattern p = Pattern.compile(literalQuery, Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
+            Pattern p = Pattern.compile(queryEscaped, Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
             final String contentStr = KeywordSearch.getServer().getCore().getSolrContent(f);
             Matcher m = p.matcher(contentStr);
             if (m.find()) {
@@ -174,6 +197,7 @@ public class TermComponentQuery implements KeywordSearchQuery {
             }
         }
 
+
         return results;
     }
 
@@ -219,7 +243,7 @@ public class TermComponentQuery implements KeywordSearchQuery {
             rootNode = Node.EMPTY;
         }
 
-        final String pathText = "RegEx query";
+        final String pathText = "Term query";
         // String pathText = "RegEx query: " + termsQuery
         //+ "    Files with exact matches: " + Long.toString(totalMatches) + " (also listing approximate matches)";
 
@@ -243,18 +267,10 @@ public class TermComponentQuery implements KeywordSearchQuery {
             progress.start();
             progress.progress("Running Terms query.");
 
-            List<Term> terms = executeQuery(q);
+            terms = executeQuery(q);
 
             progress.progress("Terms query completed.");
 
-            //debug query
-            //StringBuilder sb = new StringBuilder();
-            //for (Term t : terms) {
-            //    sb.append(t.getTerm() + " : " + t.getFrequency() + "\n");
-            //}
-            //logger.log(Level.INFO, "TermsComponent query result: " + sb.toString());
-            //end debug query
-
             return terms;
         }