Separate hashset hits in directory tree, various fixes

DataModel/src/org/sleuthkit/autopsy/datamodel/DisplayableItemNodeVisitor.java

@@ -19,6 +19,7 @@
 package org.sleuthkit.autopsy.datamodel;
 
 import org.sleuthkit.autopsy.datamodel.HashsetHits.HashsetHitsRootNode;
+import org.sleuthkit.autopsy.datamodel.HashsetHits.HashsetHitsSetNode;
 import org.sleuthkit.autopsy.datamodel.KeywordHits.KeywordHitsKeywordNode;
 import org.sleuthkit.autopsy.datamodel.KeywordHits.KeywordHitsListNode;
 import org.sleuthkit.autopsy.datamodel.KeywordHits.KeywordHitsRootNode;
@@ -43,6 +44,7 @@ public interface DisplayableItemNodeVisitor<T> {
     T visit(KeywordHitsListNode khsn);
     T visit(KeywordHitsKeywordNode khmln);
     T visit(HashsetHitsRootNode hhrn);
+    T visit(HashsetHitsSetNode hhsn);
     T visit(ViewsNode vn);
     T visit(ResultsNode rn);
     T visit(ImagesNode in);
@@ -150,5 +152,10 @@ public interface DisplayableItemNodeVisitor<T> {
         public T visit(HashsetHitsRootNode hhrn) {
             return defaultVisit(hhrn);
         }
+        
+        @Override
+        public T visit(HashsetHitsSetNode hhsn) {
+            return defaultVisit(hhsn);
+        }
     }
 }

DataModel/src/org/sleuthkit/autopsy/datamodel/HashsetHits.java

@@ -18,8 +18,15 @@
  */
 package org.sleuthkit.autopsy.datamodel;
 
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
 import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import org.openide.nodes.AbstractNode;
@@ -29,6 +36,7 @@ import org.openide.nodes.Node;
 import org.openide.nodes.Sheet;
 import org.openide.util.lookup.Lookups;
 import org.sleuthkit.datamodel.BlackboardArtifact;
+import org.sleuthkit.datamodel.BlackboardAttribute;
 import org.sleuthkit.datamodel.SleuthkitCase;
 import org.sleuthkit.datamodel.TskException;
 
@@ -43,9 +51,36 @@ public class HashsetHits implements AutopsyVisitableItem {
     private static final Logger logger = Logger.getLogger(HashsetHits.class.getName());
     
     private SleuthkitCase skCase;
+    private Map<String, Set<Long>> hashSetHitsMap;
     
     public HashsetHits(SleuthkitCase skCase) {
         this.skCase = skCase;
+        hashSetHitsMap = new LinkedHashMap<String, Set<Long>>();
+    }
+    
+    private void initArtifacts() {
+        hashSetHitsMap.clear();
+        try {
+            int setNameId = BlackboardAttribute.ATTRIBUTE_TYPE.TSK_HASHSET_NAME.getTypeID();
+            String query = "select value_text,artifact_id,attribute_type_id from blackboard_attributes where " + 
+                    "attribute_type_id=" + setNameId;
+            ResultSet rs = skCase.runQuery(query);
+            while(rs.next()){
+                String value = rs.getString("value_text");
+                long artifactId = rs.getLong("artifact_id");
+                if(!hashSetHitsMap.containsKey(value)) {
+                    hashSetHitsMap.put(value, new HashSet<Long>());
+                }
+                hashSetHitsMap.get(value).add(artifactId);
+                
+            }
+            Statement s = rs.getStatement();
+            rs.close();
+            if (s != null)
+                s.close();
+        } catch (SQLException ex) {
+            logger.log(Level.WARNING, "SQL Exception occurred: ", ex);
+        }
     }
 
     @Override
@@ -58,14 +93,9 @@ public class HashsetHits implements AutopsyVisitableItem {
         public HashsetHitsRootNode() {
             super(Children.create(new HashsetHitsRootChildren(), true), Lookups.singleton(DISPLAY_NAME));
             super.setName(HASHSET_HITS);
-            List<BlackboardArtifact> arts = new ArrayList<BlackboardArtifact>();
+            super.setDisplayName(DISPLAY_NAME);
-            try {
-                arts = skCase.getBlackboardArtifacts(BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT.getTypeID());
-            } catch (TskException ex) {
-                logger.log(Level.WARNING, "Error retrieving artifacts", ex);
-            }
-            super.setDisplayName(DISPLAY_NAME + " (" + arts.size() + ")");
             this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/hashset_hits.png");
+            initArtifacts();
         }
         
         @Override
@@ -91,22 +121,77 @@ public class HashsetHits implements AutopsyVisitableItem {
         }
     }
     
-    private class HashsetHitsRootChildren extends ChildFactory<BlackboardArtifact> {
+    private class HashsetHitsRootChildren extends ChildFactory<String> {
+
+        @Override
+        protected boolean createKeys(List<String> list) {
+            list.addAll(hashSetHitsMap.keySet());
+            return true;
+        }
+        
+        @Override
+        protected Node createNodeForKey(String key) {
+            return new HashsetHitsSetNode(key, hashSetHitsMap.get(key));
+        }
+    }
+    
+    public class HashsetHitsSetNode extends AbstractNode implements DisplayableItemNode {
+        
+        public HashsetHitsSetNode(String name, Set<Long> children) {
+            super(Children.create(new HashsetHitsSetChildren(children), true), Lookups.singleton(name));
+            super.setName(name);
+            super.setDisplayName(name + " (" + children.size() + ")");
+            this.setIconBaseWithExtension("org/sleuthkit/autopsy/images/hashset_hits.png");
+        }
+        
+        @Override
+        protected Sheet createSheet() {
+            Sheet s = super.createSheet();
+            Sheet.Set ss = s.get(Sheet.PROPERTIES);
+            if (ss == null) {
+                ss = Sheet.createPropertiesSet();
+                s.put(ss);
+            }
+
+            ss.put(new NodeProperty("Name",
+                    "Name",
+                    "no description",
+                    getName()));
+            
+            return s;
+        }
+
+        @Override
+        public <T> T accept(DisplayableItemNodeVisitor<T> v) {
+            return v.visit(this);
+        }
+    }
+    
+    private class HashsetHitsSetChildren extends ChildFactory<BlackboardArtifact> {
+
+        private Set<Long> children;
+
+        private HashsetHitsSetChildren(Set<Long> children) {
+            super();
+            this.children = children;
+        }
 
         @Override
         protected boolean createKeys(List<BlackboardArtifact> list) {
+            for (long l : children) {
                 try {
-                list.addAll(skCase.getBlackboardArtifacts(BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT.getTypeID()));
+                    //TODO: bulk artifact gettings
+                    list.add(skCase.getBlackboardArtifact(l));
                 } catch (TskException ex) {
-                logger.log(Level.WARNING, "Error getting Blackboard Artifacts", ex);
+                    logger.log(Level.WARNING, "TSK Exception occurred", ex);
+                }
             }
             return true;
         }
         
         @Override
-        protected Node createNodeForKey(BlackboardArtifact key) {
+        protected Node createNodeForKey(BlackboardArtifact artifact) {
-            return new BlackboardArtifactNode(key);
+            return new BlackboardArtifactNode(artifact);
         }
     }
-    
 }

DirectoryTree/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeFilterChildren.java

@@ -32,6 +32,7 @@ import org.sleuthkit.autopsy.datamodel.ExtractedContentNode;
 import org.sleuthkit.autopsy.datamodel.FileNode;
 import org.sleuthkit.autopsy.datamodel.FileSearchFilterNode;
 import org.sleuthkit.autopsy.datamodel.HashsetHits.HashsetHitsRootNode;
+import org.sleuthkit.autopsy.datamodel.HashsetHits.HashsetHitsSetNode;
 import org.sleuthkit.autopsy.datamodel.ImagesNode;
 import org.sleuthkit.autopsy.datamodel.KeywordHits.KeywordHitsKeywordNode;
 import org.sleuthkit.autopsy.datamodel.KeywordHits.KeywordHitsListNode;
@@ -84,6 +85,7 @@ class DirectoryTreeFilterChildren extends FilterNode.Children {
                 || arg0 instanceof RecentFilesNode
                 || arg0 instanceof KeywordHitsRootNode
                 || arg0 instanceof KeywordHitsListNode
+                || arg0 instanceof HashsetHitsRootNode
                 || arg0 instanceof ImagesNode
                 || arg0 instanceof ViewsNode
                 || arg0 instanceof ResultsNode)) {
@@ -95,7 +97,7 @@ class DirectoryTreeFilterChildren extends FilterNode.Children {
                 || arg0 instanceof ArtifactTypeNode 
                 || arg0 instanceof RecentFilesFilterNode
                 || arg0 instanceof FileSearchFilterNode
-                || arg0 instanceof HashsetHitsRootNode
+                || arg0 instanceof HashsetHitsSetNode
                 )) {
             return new Node[]{this.copyNode(arg0, false)};
         } else {

DirectoryTree/src/org/sleuthkit/autopsy/directorytree/DirectoryTreeTopComponent.java

@@ -819,7 +819,20 @@ public final class DirectoryTreeTopComponent extends TopComponent implements Dat
         Node resultsNode = rootChilds.findChild(ResultsNode.NAME);
         Children resultsChilds = resultsNode.getChildren();
         if (type.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_HASHSET_HIT)) {
-            treeNode = resultsChilds.findChild(type.getLabel());
+            Node hashsetRootNode = resultsChilds.findChild(type.getLabel());
+            Children hashsetRootChilds = hashsetRootNode.getChildren();
+            try {
+                String setName = null;
+                List<BlackboardAttribute> attributes = art.getAttributes();
+                for(BlackboardAttribute att : attributes) {
+                    int typeId = att.getAttributeTypeID();
+                    if (typeId == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_HASHSET_NAME.getTypeID())
+                        setName = att.getValueString();
+                }
+                treeNode = hashsetRootChilds.findChild(setName);
+            } catch (TskException ex) {
+                logger.log(Level.WARNING, "Error retrieving attributes", ex);
+            }
         } else if (type.equals(BlackboardArtifact.ARTIFACT_TYPE.TSK_KEYWORD_HIT)) {
             Node keywordRootNode = resultsChilds.findChild(type.getLabel());
             Children keywordRootChilds = keywordRootNode.getChildren();

HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestService.java

@@ -51,7 +51,8 @@ public class HashDbIngestService implements IngestServiceFsContent {
     private static int messageId = 0;
     private int count;
     // Whether or not to do hash lookups (only set to true if there are dbs set)
-    private boolean process;
+    private boolean nsrlIsSet;
+    private boolean knownBadIsSet;
     private HashDb nsrlSet;
     private int nsrlPointer;
     private Map<Integer, HashDb> knownBadSets = new HashMap<Integer, HashDb>();
@@ -77,7 +78,7 @@ public class HashDbIngestService implements IngestServiceFsContent {
     @Override
     public void init(IngestManagerProxy managerProxy) {
         HashDbMgmtPanel.getDefault().setIngestRunning(true);
-        this.process = false;
+        HashDbSimplePanel.setIngestRunning(true);
         this.managerProxy = managerProxy;
         this.managerProxy.postMessage(IngestMessage.createMessage(++messageId, IngestMessage.MessageType.INFO, this, "Started"));
         this.skCase = Case.getCurrentCase().getSleuthkitCase();
@@ -86,13 +87,12 @@ public class HashDbIngestService implements IngestServiceFsContent {
             nsrlSet = null;
             knownBadSets.clear();
             skCase.clearLookupDatabases();
-            boolean nsrlIsSet = false;
+            nsrlIsSet = false;
-            boolean knownBadIsSet = false;
+            knownBadIsSet = false;
             
             HashDb nsrl = hdbxml.getNSRLSet();
             if(nsrl != null && IndexStatus.isIngestible(nsrl.status())) {
                 nsrlIsSet = true;
-                this.process = true;
                 this.nsrlSet = nsrl;
                 nsrlPointer = skCase.setNSRLDatabase(nsrl.getDatabasePaths().get(0));
             }
@@ -100,7 +100,6 @@ public class HashDbIngestService implements IngestServiceFsContent {
             for(HashDb db : hdbxml.getKnownBadSets()) {
                 IndexStatus status = db.status();
                 if (db.getUseForIngest() && IndexStatus.isIngestible(status)) { // TODO: should inform user that we won't use the db if it's not indexed
-                    this.process = true;
                     knownBadIsSet = true;
                     int ret = skCase.addKnownBadDatabase(db.getDatabasePaths().get(0)); // TODO: support multiple paths
                     knownBadSets.put(ret, db);
@@ -150,6 +149,7 @@ public class HashDbIngestService implements IngestServiceFsContent {
         managerProxy.postMessage(IngestMessage.createMessage(++messageId, IngestMessage.MessageType.INFO, this, "Hash Ingest Complete", detailsSb.toString()));
         
         HashDbMgmtPanel.getDefault().setIngestRunning(false);
+        HashDbSimplePanel.setIngestRunning(false);
     }
 
     /**
@@ -159,6 +159,7 @@ public class HashDbIngestService implements IngestServiceFsContent {
     public void stop() {
         //manager.postMessage(IngestMessage.createMessage(++messageId, IngestMessage.MessageType.INFO, this, "STOP"));
         HashDbMgmtPanel.getDefault().setIngestRunning(false);
+        HashDbSimplePanel.setIngestRunning(false);
     }
 
     /**
@@ -185,12 +186,12 @@ public class HashDbIngestService implements IngestServiceFsContent {
     @Override
     public ProcessResult process(FsContent fsContent) {
         ProcessResult ret = ProcessResult.UNKNOWN;
-        process = true;
+        boolean processFile = true;
         if(fsContent.getKnown().equals(TskData.FileKnown.BAD)) {
             ret = ProcessResult.OK;
-            process = false;
+            processFile = false;
         }
-        if (process) {
+        if (processFile && (nsrlIsSet || knownBadIsSet)) {
             String name = fsContent.getName();
             try {
                 String md5Hash = Hash.calculateMd5(fsContent);
@@ -207,7 +208,7 @@ public class HashDbIngestService implements IngestServiceFsContent {
                     }
                     ret = ProcessResult.OK;
                 }
-                if(!foundBad) {
+                if(!foundBad && nsrlIsSet) {
                     status = skCase.nsrlLookupMd5(md5Hash);
                     if (status.equals(TskData.FileKnown.KNOWN)) {
                         skCase.setKnown(fsContent, status);

HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbMgmtPanel.java

@@ -287,12 +287,7 @@ public class HashDbMgmtPanel extends javax.swing.JPanel {
                 if (HashDb.isIndexPath(filePath)) {
                     filePath = HashDb.toDatabasePath(filePath);
                 }
-                String derivedName;
+                String derivedName = SleuthkitJNI.getDatabaseName(filePath);
-                try {
-                    derivedName = SleuthkitJNI.getDatabaseName(filePath);
-                } catch (TskException ex) {
-                    derivedName = "";
-                }
                 
                 String setName = (String) JOptionPane.showInputDialog(this, "New Hash Set name:", "New Hash Set", 
                         JOptionPane.PLAIN_MESSAGE, null, null, derivedName);
@@ -307,6 +302,15 @@ public class HashDbMgmtPanel extends javax.swing.JPanel {
 
             } catch (IOException ex) {
                 logger.log(Level.WARNING, "Couldn't get selected file path.", ex);
+            } catch (TskException ex) {
+                logger.log(Level.WARNING, "Invalid database: ", ex);
+                int tryAgain = JOptionPane.showConfirmDialog(this, 
+                            "Database file you chose cannot be opened.\n" + 
+                            "If it was just an index, please try to recreate it from the database.\n" + 
+                            "Would you like to choose another database?", "Invalid File", JOptionPane.YES_NO_OPTION);
+                if(tryAgain == JOptionPane.YES_OPTION) {
+                    setNSRLButtonActionPerformed(null);
+                }
             }
         }
         save();
@@ -331,12 +335,7 @@ public class HashDbMgmtPanel extends javax.swing.JPanel {
                 if (HashDb.isIndexPath(filePath)) {
                     filePath = HashDb.toDatabasePath(filePath);
                 }
-                String derivedName;
+                String derivedName = SleuthkitJNI.getDatabaseName(filePath);
-                try {
-                    derivedName = SleuthkitJNI.getDatabaseName(filePath);
-                } catch (TskException ex) {
-                    derivedName = "";
-                }
 
                 this.nsrlSet = new HashDb(derivedName, Arrays.asList(new String[]{filePath}), false); // TODO: support multiple file paths
                 int toIndex = JOptionPane.NO_OPTION;
@@ -357,11 +356,17 @@ public class HashDbMgmtPanel extends javax.swing.JPanel {
                 if(toIndex == JOptionPane.YES_OPTION) {
                     indexNSRLButtonActionPerformed(null);
                 }
-
-                
-                
             } catch (IOException ex) {
                 logger.log(Level.WARNING, "Couldn't get selected file path.", ex);
+            } catch (TskException ex) {
+                logger.log(Level.WARNING, "Invalid database: ", ex);
+                int tryAgain = JOptionPane.showConfirmDialog(this, 
+                            "Database file you chose cannot be opened.\n" + 
+                            "If it was just an index, please try to recreate it from the database.\n" + 
+                            "Would you like to choose another database?", "Invalid File", JOptionPane.YES_NO_OPTION);
+                if(tryAgain == JOptionPane.YES_OPTION) {
+                    setNSRLButtonActionPerformed(null);
+                }
             }
         }
     }//GEN-LAST:event_setNSRLButtonActionPerformed
@@ -468,7 +473,7 @@ public class HashDbMgmtPanel extends javax.swing.JPanel {
                     if(((Boolean) getValueAt(rowIndex, columnIndex)) || IndexStatus.isIngestible(entry.status()))
                         entry.setUseForIngest((Boolean) aValue);
                     else
-                        JOptionPane.showMessageDialog(HashDbMgmtPanel.this, "Databases must be indexed before they can be used for ingest.");
+                        JOptionPane.showMessageDialog(HashDbMgmtPanel.this, "Databases must be indexed before they can be used for ingest");
             }
         }
 
@@ -520,11 +525,6 @@ public class HashDbMgmtPanel extends javax.swing.JPanel {
     }
     
     static void setButtonFromIndexStatus(JButton theButton, IndexStatus status) {
-        if(ingestRunning) {
-            theButton.setText("Not Available");
-            theButton.setEnabled(false);
-            return;
-        }
         switch (status) {
             case INDEX_OUTDATED:
                 theButton.setText("Re-index");
@@ -546,6 +546,9 @@ public class HashDbMgmtPanel extends javax.swing.JPanel {
                 theButton.setText("No DB");
                 theButton.setEnabled(false);
         }
+        if (ingestRunning) {
+            theButton.setEnabled(false);
+        }
     }
 }
 

HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbSimplePanel.java

@@ -27,6 +27,7 @@ package org.sleuthkit.autopsy.hashdatabase;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.logging.Logger;
+import javax.swing.JOptionPane;
 import javax.swing.table.AbstractTableModel;
 import javax.swing.table.TableColumn;
 
@@ -39,6 +40,7 @@ public class HashDbSimplePanel extends javax.swing.JPanel {
     private static final Logger logger = Logger.getLogger(HashDbSimplePanel.class.getName());
     private HashTableModel knownBadTableModel;
     private HashDb nsrl;
+    private static boolean ingestRunning = false;
 
     /** Creates new form HashDbSimplePanel */
     public HashDbSimplePanel() {
@@ -47,6 +49,10 @@ public class HashDbSimplePanel extends javax.swing.JPanel {
         customizeComponents();
     }
     
+    static void setIngestRunning(boolean running) {
+        ingestRunning = running;
+    }
+    
     private void customizeComponents() {
         notableHashTable.setModel(knownBadTableModel);
         
@@ -174,14 +180,18 @@ public class HashDbSimplePanel extends javax.swing.JPanel {
         
         @Override
         public boolean isCellEditable(int rowIndex, int columnIndex) {
-            return columnIndex == 0;
+            return !ingestRunning && columnIndex == 0;
         }
 
         @Override
         public void setValueAt(Object aValue, int rowIndex, int columnIndex) {
             if(columnIndex == 0){
                 HashDb db = xmlHandle.getKnownBadSets().get(rowIndex);
+                if(((Boolean) getValueAt(rowIndex, columnIndex)) || IndexStatus.isIngestible(db.status())) {
                         db.setUseForIngest((Boolean) aValue);
+                } else {
+                        JOptionPane.showMessageDialog(HashDbSimplePanel.this, "Databases must be indexed before they can be used for ingest");
+                }
             }
         }