From 69d55f7cc174f15bb09195dd5e7c6f268a1e6f6c Mon Sep 17 00:00:00 2001
From: adam-m <amalinowski@basistech.com>
Date: Tue, 27 Mar 2012 00:16:28 -0400
Subject: [PATCH] Keyword search ingest improvements: alternative
 performQueryPerTerm() and equivalent bb write methods - more optimal for
 grouping results per hit (in ingest) rather then per query. Improves overall
 keyword search ingest performance. Improve regex results highlight by using
 content_ws field.

---
 .../release/solr/solr/conf/schema.xml         |   2 +-
 .../KeywordSearchIngestService.java           | 102 ++++++-----
 .../keywordsearch/KeywordSearchQuery.java     |  21 +++
 .../KeywordSearchQueryManager.java            |  25 ++-
 .../autopsy/keywordsearch/LuceneQuery.java    |  40 ++++-
 .../keywordsearch/TermComponentQuery.java     | 163 +++++++++++-------
 6 files changed, 229 insertions(+), 124 deletions(-)

diff --git a/KeywordSearch/release/solr/solr/conf/schema.xml b/KeywordSearch/release/solr/solr/conf/schema.xml
index 0436a567a9..abdd660793 100644
--- a/KeywordSearch/release/solr/solr/conf/schema.xml
+++ b/KeywordSearch/release/solr/solr/conf/schema.xml
@@ -534,7 +534,7 @@
    
    <!-- field with white-space tokenized words for TermsComponent regex search (useful for fast search of IP addresses, URLs, certain phone numbers)
 		also be useful for Lucene based queries containing special characters-->
-   <field name="content_ws" type="text_ws" indexed="true" stored="false" multiValued="true"/> 
+   <field name="content_ws" type="text_ws" indexed="true" stored="true" multiValued="true"/> 
 	
    <!-- Uncommenting the following will create a "timestamp" field using
         a default value of "NOW" to indicate when each document was indexed.
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestService.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestService.java
index 897c1f60e5..718ffed74c 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestService.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestService.java
@@ -57,7 +57,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
     public static final String MODULE_NAME = "Keyword Search";
     private static KeywordSearchIngestService instance = null;
     private IngestManagerProxy managerProxy;
-    private static final long MAX_STRING_EXTRACT_SIZE = 10 * (1 << 10); // * (1 << 10);
+    private static final long MAX_STRING_EXTRACT_SIZE = 1 * (1 << 10); // * (1 << 10); TODO increase
     private static final long MAX_INDEX_SIZE = 100 * (1 << 10) * (1 << 10);
     private Ingester ingester;
     private volatile boolean commitIndex = false; //whether to commit index next time
@@ -78,7 +78,6 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
     private volatile boolean finalRunComplete = false;
     private final String hashDBServiceName = "Hash Lookup";
     private SleuthkitCase caseHandle = null;
-  
     // TODO: use a more robust method than checking file extension to determine
     // whether to try a file
     // supported extensions list from http://www.lucidimagination.com/devzone/technical-articles/content-extraction-tika
@@ -89,8 +88,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
 
     public enum IngestStatus {
 
-        INGESTED, EXTRACTED_INGESTED, SKIPPED,
-    };
+        INGESTED, EXTRACTED_INGESTED, SKIPPED,};
     private Map<Long, IngestStatus> ingestStatus;
     private Map<String, List<FsContent>> reportedHits; //already reported hits
 
@@ -109,14 +107,14 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
         //logger.log(Level.INFO, "hashdb result: " + hashDBResult + "file: " + fsContent.getName());
         if (hashDBResult == IngestServiceFsContent.ProcessResult.COND_STOP) {
             return ProcessResult.OK;
-        }
-        else if (hashDBResult == IngestServiceFsContent.ProcessResult.ERROR) {
+        } else if (hashDBResult == IngestServiceFsContent.ProcessResult.ERROR) {
             //notify depending service that keyword search (would) encountered error for this file
             return ProcessResult.ERROR;
         }
-        
-        if (processedFiles == false)
+
+        if (processedFiles == false) {
             processedFiles = true;
+        }
 
         //check if time to commit and previous search is not running
         //commiting while searching causes performance issues
@@ -201,7 +199,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
         caseHandle = Case.getCurrentCase().getSleuthkitCase();
 
         this.managerProxy = managerProxy;
-        
+
         //this deregisters previously registered listeners at every init()
         pcs = new PropertyChangeSupport(KeywordSearchIngestService.class);
 
@@ -286,16 +284,16 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
         //no need to check timer thread
 
     }
-    
+
     @Override
     public synchronized boolean backgroundJobsCompleteListener(PropertyChangeListener l) {
-        if (finalRunComplete == true)
+        if (finalRunComplete == true) {
             return false;
-        else {
+        } else {
             pcs.addPropertyChangeListener(l);
             return true;
         }
-       
+
     }
 
     private void commit() {
@@ -504,7 +502,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
         }
     }
 
-    private class Searcher extends SwingWorker<Object,Void> {
+    private class Searcher extends SwingWorker<Object, Void> {
 
         private List<Keyword> keywords;
         private ProgressHandle progress;
@@ -547,52 +545,69 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
 
                 KeywordSearchQuery del = null;
 
-                if (keywordQuery.isLiteral()) {
+                boolean isRegex = !keywordQuery.isLiteral();
+                if (!isRegex) {
                     del = new LuceneQuery(keywordQuery);
                     del.escape();
                 } else {
                     del = new TermComponentQuery(keywordQuery);
                 }
 
-                List<FsContent> queryResult = null;
+                Map<String, List<FsContent>> queryResult = null;
 
                 try {
-                    queryResult = del.performQuery();
+                    queryResult = del.performQueryPerTerm();
                 } catch (Exception e) {
                     logger.log(Level.INFO, "Error performing query: " + keywordQuery.getQuery(), e);
                     continue;
                 }
 
                 //calculate new results but substracting results already obtained in this run
-                List<FsContent> newResults = new ArrayList<FsContent>();
+                Map<Keyword, List<FsContent>> newResults = new HashMap<Keyword, List<FsContent>>();
 
-                List<FsContent> curResults = currentResults.get(keywordQuery);
-                if (curResults == null) {
-                    currentResults.put(keywordQuery, queryResult);
-                    newResults = queryResult;
-                } else {
-                    for (FsContent res : queryResult) {
-                        if (!curResults.contains(res)) {
-                            //add to new results
-                            newResults.add(res);
+                for (String termResult : queryResult.keySet()) {
+                    List<FsContent> queryTermResults = queryResult.get(termResult);
+                    Keyword termResultK = new Keyword(termResult, !isRegex);
+                    List<FsContent> curTermResults = currentResults.get(termResultK);
+                    if (curTermResults == null) {
+                        currentResults.put(termResultK, queryTermResults);
+                        newResults.put(termResultK, queryTermResults);
+                    } else {
+                        //some fscontent hits already exist for this keyword
+                        for (FsContent res : queryTermResults) {
+                            if (!curTermResults.contains(res)) {
+                                //add to new results
+                                List<FsContent> newResultsFs = newResults.get(termResultK);
+                                if (newResultsFs == null) {
+                                    newResultsFs = new ArrayList<FsContent>();
+                                    newResults.put(termResultK, newResultsFs);
+                                }
+                                newResultsFs.add(res);
+                                curTermResults.add(res);
+                            }
                         }
                     }
-                    //update current result with new ones
-                    curResults.addAll(newResults);
 
                 }
 
+
                 if (!newResults.isEmpty()) {
 
                     //write results to BB
                     Collection<BlackboardArtifact> newArtifacts = new ArrayList<BlackboardArtifact>(); //new artifacts to report
-                    for (FsContent hitFile : newResults) {
-                        if (this.isCancelled()) {
-                            return null;
-                        }
-                        Collection<KeywordWriteResult> written = del.writeToBlackBoard(hitFile, listName);
-                        for (KeywordWriteResult res : written) {
-                            newArtifacts.add(res.getArtifact());
+                    for (final Keyword hitTerm : newResults.keySet()) {
+                        List<FsContent> fsContentHits = newResults.get(hitTerm);
+                        for (final FsContent hitFile : fsContentHits) {
+                            if (this.isCancelled()) {
+                                return null;
+                            }
+                            KeywordWriteResult written = del.writeToBlackBoard(hitTerm.getQuery(), hitFile, listName);
+                            if (written == null) {
+                                logger.log(Level.INFO, "BB artifact for keyword not written: " + hitTerm.toString());
+                                continue;
+                            }
+
+                            newArtifacts.add(written.getArtifact());
 
                             //generate a data message for each artifact
                             StringBuilder subjectSb = new StringBuilder();
@@ -606,7 +621,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
                             }
                             subjectSb.append("<");
                             String uniqueKey = null;
-                            BlackboardAttribute attr = res.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID());
+                            BlackboardAttribute attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID());
                             if (attr != null) {
                                 final String keyword = attr.getValueString();
                                 subjectSb.append(keyword);
@@ -625,7 +640,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
                             detailsSb.append("</tr>");
 
                             //preview
-                            attr = res.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID());
+                            attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID());
                             if (attr != null) {
                                 detailsSb.append("<tr>");
                                 detailsSb.append("<th>Preview</th>");
@@ -642,7 +657,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
 
 
                             //list
-                            attr = res.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SET.getTypeID());
+                            attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_SET.getTypeID());
                             detailsSb.append("<tr>");
                             detailsSb.append("<th>List</th>");
                             detailsSb.append("<td>").append(attr.getValueString()).append("</td>");
@@ -650,7 +665,7 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
 
                             //regex
                             if (!keywordQuery.isLiteral()) {
-                                attr = res.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getTypeID());
+                                attr = written.getAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getTypeID());
                                 if (attr != null) {
                                     detailsSb.append("<tr>");
                                     detailsSb.append("<th>RegEx</th>");
@@ -661,9 +676,10 @@ public final class KeywordSearchIngestService implements IngestServiceFsContent
                             }
                             detailsSb.append("</table>");
 
-                            managerProxy.postMessage(IngestMessage.createDataMessage(++messageID, instance, subjectSb.toString(), detailsSb.toString(), uniqueKey, res.getArtifact()));
-                        }
-                    } //for each file hit
+                            managerProxy.postMessage(IngestMessage.createDataMessage(++messageID, instance, subjectSb.toString(), detailsSb.toString(), uniqueKey, written.getArtifact()));
+
+                        } //for each term hit
+                    }//for each file hit
 
                     //update artifact browser
                     IngestManager.fireServiceDataEvent(new ServiceDataEvent(MODULE_NAME, ARTIFACT_TYPE.TSK_KEYWORD_HIT, newArtifacts));
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQuery.java
index 1a716d6451..92ffbccf23 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQuery.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQuery.java
@@ -20,6 +20,7 @@ package org.sleuthkit.autopsy.keywordsearch;
 
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 import org.apache.solr.client.solrj.response.TermsResponse.Term;
 import org.sleuthkit.datamodel.FsContent;
 
@@ -34,10 +35,20 @@ public interface KeywordSearchQuery {
     
     /**
      * execute query and return results without publishing them
+     * return results for all matching terms
+     * 
      * @return 
      */
     public List<FsContent> performQuery();
     
+    /**
+     * execute query and return results without publishing them
+     * return results per term
+     * 
+     * @return 
+     */
+    public Map<String,List<FsContent>> performQueryPerTerm();
+    
     
     /**
      * execute the query and publish results
@@ -80,6 +91,16 @@ public interface KeywordSearchQuery {
      * @return collection of results (with cached bb artifacts/attributes) created and written
      */
     public Collection<KeywordWriteResult> writeToBlackBoard(FsContent newFsHit, String listName);
+    
+    /**
+     * write results to blackboard per single term and file hit
+     * this method is useful if something else should keep track of partial results to write
+     * @param termHit term for only which to write results
+     * @param newFsHit fscontent for which to write results for this hit
+     * @param listName listname
+     * @return collection of results (with cached bb artifacts/attributes) created and written
+     */
+    public KeywordWriteResult writeToBlackBoard(String termHit, FsContent newFsHit, String listName);
        
 }
 
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQueryManager.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQueryManager.java
index 7ebdf02a89..7acb9b7faa 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQueryManager.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchQueryManager.java
@@ -35,6 +35,7 @@ import org.sleuthkit.autopsy.datamodel.KeyValue;
 import org.sleuthkit.autopsy.keywordsearch.KeywordSearch.QueryType;
 import org.sleuthkit.datamodel.FsContent;
 
+
 /**
  * Query manager responsible for running appropriate queries and displaying results
  * for single, multi keyword queries, with detailed or collapsed results
@@ -143,10 +144,16 @@ public class KeywordSearchQueryManager implements KeywordSearchQuery {
 
     @Override
     public List<FsContent> performQuery() {
-        //not done here
-        return null;
+        throw new UnsupportedOperationException("performQuery() unsupported");
     }
 
+    @Override
+    public Map<String, List<FsContent>> performQueryPerTerm() {
+        throw new UnsupportedOperationException("performQueryPerTerm() unsupported");
+    }
+    
+    
+
     @Override
     public boolean validate() {
         boolean allValid = true;
@@ -192,14 +199,16 @@ public class KeywordSearchQueryManager implements KeywordSearchQuery {
     
     @Override
     public Collection<KeywordWriteResult> writeToBlackBoard(FsContent newFsHit, String listName) {     
-        Collection<KeywordWriteResult> ret = new ArrayList<KeywordWriteResult>();
-         for (KeywordSearchQuery q : queryDelegates) {
-            ret.addAll(q.writeToBlackBoard(newFsHit, listName));
-        }
-        return ret;
+        throw new UnsupportedOperationException("writeToBlackBoard() unsupported by manager");
     }
-}
 
+    @Override
+    public KeywordWriteResult writeToBlackBoard(String termHit, FsContent newFsHit, String listName) {
+        throw new UnsupportedOperationException("writeToBlackBoard() unsupported by manager");
+    }
+    
+    
+}
 /**
  * custom KeyValue that also stores query object  to execute
  */
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java
index e7d6ce59c7..817a2fad5c 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/LuceneQuery.java
@@ -23,6 +23,7 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.logging.Level;
@@ -59,9 +60,8 @@ public class LuceneQuery implements KeywordSearchQuery {
     private Keyword keywordQuery = null;
     //use different highlight Solr fields for regex and literal search
     static final String HIGHLIGHT_FIELD_LITERAL = "content";
-    //TODO change to content_ws and in Solr schema to stored="true" to improve regex highlight matching
-    static final String HIGHLIGHT_FIELD_REGEX = "content";
-    //static final String HIGHLIGHT_FIELD_REGEX = "content_ws";
+    //static final String HIGHLIGHT_FIELD_REGEX = "content";
+    static final String HIGHLIGHT_FIELD_REGEX = "content_ws";
 
     public LuceneQuery(Keyword keywordQuery) {
         this(keywordQuery.getQuery());
@@ -160,6 +160,18 @@ public class LuceneQuery implements KeywordSearchQuery {
         return matches;
     }
 
+    @Override
+    public Map<String, List<FsContent>> performQueryPerTerm() {
+        Map<String, List<FsContent>> results = new HashMap<String, List<FsContent>>();
+        //in case of single term literal query there is only 1 term, so delegate to performQuery()
+        results.put(query, performQuery());
+        
+        return results;
+    }
+    
+    
+    
+
     @Override
     public void execute() {
         escape();
@@ -211,19 +223,28 @@ public class LuceneQuery implements KeywordSearchQuery {
 
     @Override
     public Collection<KeywordWriteResult> writeToBlackBoard(FsContent newFsHit, String listName) {
+        List<KeywordWriteResult> ret = new ArrayList<KeywordWriteResult>();
+        KeywordWriteResult written = writeToBlackBoard(query, newFsHit, listName);
+        if (written != null)
+            ret.add(written);
+        return ret;
+    }
+    
+    
+
+    @Override
+    public KeywordWriteResult writeToBlackBoard(String termHit, FsContent newFsHit, String listName) {
         final String MODULE_NAME = KeywordSearchIngestService.MODULE_NAME;
 
-        Collection<KeywordWriteResult> writeResults = new ArrayList<KeywordWriteResult>();
         KeywordWriteResult writeResult = null;
         Collection<BlackboardAttribute> attributes = new ArrayList<BlackboardAttribute>();
         BlackboardArtifact bba = null;
         try {
             bba = newFsHit.newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT);
             writeResult = new KeywordWriteResult(bba);
-            writeResults.add(writeResult);
         } catch (Exception e) {
             logger.log(Level.INFO, "Error adding bb artifact for keyword hit", e);
-            return writeResults;
+            return null;
         }
 
         String snippet = null;
@@ -236,7 +257,7 @@ public class LuceneQuery implements KeywordSearchQuery {
             attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID(), MODULE_NAME, "", KeywordSearchUtil.escapeForBlackBoard(snippet)));
         }
         //keyword
-        attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID(), MODULE_NAME, "", query));
+        attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID(), MODULE_NAME, "", termHit));
         //list
         if (listName == null) {
             listName = "";
@@ -249,17 +270,18 @@ public class LuceneQuery implements KeywordSearchQuery {
         if (keywordQuery != null) {
             BlackboardAttribute.ATTRIBUTE_TYPE selType = keywordQuery.getType();
             if (selType != null) {
-                attributes.add(new BlackboardAttribute(selType.getTypeID(), MODULE_NAME, "", query));
+                attributes.add(new BlackboardAttribute(selType.getTypeID(), MODULE_NAME, "", termHit));
             }
         }
 
         try {
             bba.addAttributes(attributes); //write out to bb
             writeResult.add(attributes);
+            return writeResult;
         } catch (TskException e) {
             logger.log(Level.INFO, "Error adding bb attributes to artifact", e);
         }
-        return writeResults;
+        return null;
     }
 
     /**
diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java
index bb41384b85..3c114efd08 100644
--- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java
+++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/TermComponentQuery.java
@@ -18,8 +18,6 @@
  */
 package org.sleuthkit.autopsy.keywordsearch;
 
-import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -30,7 +28,6 @@ import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 import javax.swing.SwingWorker;
@@ -156,9 +153,73 @@ public class TermComponentQuery implements KeywordSearchQuery {
     }
 
     @Override
-    public Collection<KeywordWriteResult> writeToBlackBoard(FsContent newFsHit, String listName) {
+    public KeywordWriteResult writeToBlackBoard(String termHit, FsContent newFsHit, String listName) {
         final String MODULE_NAME = KeywordSearchIngestService.MODULE_NAME;
 
+        //snippet
+        String snippet = null;
+        try {
+            snippet = LuceneQuery.querySnippet(KeywordSearchUtil.escapeLuceneQuery(termHit, true, false), newFsHit.getId(), true);
+        } catch (Exception e) {
+            logger.log(Level.INFO, "Error querying snippet: " + termHit, e);
+            return null;
+        }
+
+        if (snippet == null || snippet.equals("")) {
+            return null;
+        }
+
+        //there is match actually in this file, create artifact only then
+        BlackboardArtifact bba = null;
+        KeywordWriteResult writeResult = null;
+        Collection<BlackboardAttribute> attributes = new ArrayList<BlackboardAttribute>();
+        try {
+            bba = newFsHit.newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT);
+            writeResult = new KeywordWriteResult(bba);
+        } catch (Exception e) {
+            logger.log(Level.INFO, "Error adding bb artifact for keyword hit", e);
+            return null;
+        }
+
+
+        //regex match
+        attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID(), MODULE_NAME, "", termHit));
+        //list
+        if (listName == null) {
+            listName = "";
+        }
+        attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SET.getTypeID(), MODULE_NAME, "", listName));
+
+        //preview
+        attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID(), MODULE_NAME, "", snippet));
+
+        //regex keyword
+        attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getTypeID(), MODULE_NAME, "", termsQuery));
+
+        //selector TODO move to general info artifact
+            /*
+        if (keywordQuery != null) {
+        BlackboardAttribute.ATTRIBUTE_TYPE selType = keywordQuery.getType();
+        if (selType != null) {
+        BlackboardAttribute selAttr = new BlackboardAttribute(selType.getTypeID(), MODULE_NAME, "", regexMatch);
+        attributes.add(selAttr);
+        }
+        } */
+
+        try {
+            bba.addAttributes(attributes);
+            writeResult.add(attributes);
+            return writeResult;
+        } catch (TskException e) {
+            logger.log(Level.INFO, "Error adding bb attributes for terms search artifact", e);
+        }
+        
+        return null;
+
+    }
+
+    @Override
+    public Collection<KeywordWriteResult> writeToBlackBoard(FsContent newFsHit, String listName) {
         Collection<KeywordWriteResult> writeResults = new ArrayList<KeywordWriteResult>();
 
         //get unique term matches, all cases
@@ -169,70 +230,46 @@ public class TermComponentQuery implements KeywordSearchQuery {
         }
 
         for (String regexMatch : matches.keySet()) {
-            //snippet
-            String snippet = null;
-            try {
-                snippet = LuceneQuery.querySnippet(KeywordSearchUtil.escapeLuceneQuery(regexMatch, true, false), newFsHit.getId(), true);
-            } catch (Exception e) {
-                logger.log(Level.INFO, "Error querying snippet: " + regexMatch, e);
-                continue;
-            }
-
-            if (snippet == null || snippet.equals("")) {
-                continue;
-            }
-
-            //there is match actually in this file, create artifact only then
-            BlackboardArtifact bba = null;
-            KeywordWriteResult writeResult = null;
-            Collection<BlackboardAttribute> attributes = new ArrayList<BlackboardAttribute>();
-            try {
-                bba = newFsHit.newArtifact(ARTIFACT_TYPE.TSK_KEYWORD_HIT);
-                writeResult = new KeywordWriteResult(bba);
-                writeResults.add(writeResult);
-            } catch (Exception e) {
-                logger.log(Level.INFO, "Error adding bb artifact for keyword hit", e);
-                continue;
-            }
-
-
-            //regex match
-            attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD.getTypeID(), MODULE_NAME, "", regexMatch));
-            //list
-            if (listName == null) {
-                listName = "";
-            }
-            attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_SET.getTypeID(), MODULE_NAME, "", listName));
-
-            //preview
-            attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_PREVIEW.getTypeID(), MODULE_NAME, "", snippet));
-
-            //regex keyword
-            attributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_KEYWORD_REGEXP.getTypeID(), MODULE_NAME, "", termsQuery));
-
-            //selector TODO move to general info artifact
-            /*
-            if (keywordQuery != null) {
-                BlackboardAttribute.ATTRIBUTE_TYPE selType = keywordQuery.getType();
-                if (selType != null) {
-                    BlackboardAttribute selAttr = new BlackboardAttribute(selType.getTypeID(), MODULE_NAME, "", regexMatch);
-                    attributes.add(selAttr);
-                }
-            } */
-
-            try {
-                bba.addAttributes(attributes);
-                writeResult.add(attributes);
-            } catch (TskException e) {
-                logger.log(Level.INFO, "Error adding bb attributes for terms search artifact", e);
-            }
-
-
+            KeywordWriteResult written = writeToBlackBoard(regexMatch, newFsHit, listName);
+            if (written != null)
+                writeResults.add(written);
         } //for each term
 
         return writeResults;
     }
 
+    @Override
+    public Map<String, List<FsContent>> performQueryPerTerm() {
+        Map<String, List<FsContent>> results = new HashMap<String, List<FsContent>>();
+
+        final SolrQuery q = createQuery();
+        terms = executeQuery(q);
+
+
+        for (Term term : terms) {
+            final String termS = KeywordSearchUtil.escapeLuceneQuery(term.getTerm(), true, false);
+            if (termS.contains("*")) {
+                continue;
+            }
+
+            StringBuilder filesQueryB = new StringBuilder();
+            filesQueryB.append(TERMS_SEARCH_FIELD).append(":").append(termS);
+            final String queryStr = filesQueryB.toString();
+
+            LuceneQuery filesQuery = new LuceneQuery(queryStr);
+            try {
+                List<FsContent> subResults = filesQuery.performQuery();
+                results.put(term.getTerm(), subResults);
+            } catch (RuntimeException e) {
+                logger.log(Level.SEVERE, "Error executing Solr query,", e);
+            }
+
+        }
+
+
+        return results;
+    }
+
     /**
      * return collapsed matches with all files for the query
      * without per match breakdown